diff --git a/contrib/one-true-awk/FIXES b/contrib/one-true-awk/FIXES index 8a2befccd624..a043b356fafa 100644 --- a/contrib/one-true-awk/FIXES +++ b/contrib/one-true-awk/FIXES @@ -1,1375 +1,108 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ -This file lists all bug fixes, changes, etc., made since the AWK book -was sent to the printers in August, 1987. +This file lists all bug fixes, changes, etc., made since the +second edition of the AWK book was published in September 2023. -July 27, 2021: - As per IEEE Std 1003.1-2008, -F "str" is now consistent with - -v FS="str" when str is null. Thanks to Warner Losh. - -July 24, 2021: - Fix readrec's definition of a record. This fixes an issue - with NetBSD's RS regular expression support that can cause - an infinite read loop. Thanks to Miguel Pineiro Jr. - - Fix regular expression RS ^-anchoring. RS ^-anchoring needs to - know if it is reading the first record of a file. This change - restores a missing line that was overlooked when porting NetBSD's - RS regex functionality. Thanks to Miguel Pineiro Jr. - - Fix size computation in replace_repeat() for special case - REPEAT_WITH_Q. Thanks to Todd C. Miller. - -February 15, 2021: - Small fix so that awk will compile again with g++. Thanks to +Jan 22, 2024: + Restore the ability to compile with g++. Thanks to Arnold Robbins. -January 06, 2021: - Fix a decision bug with trailing stuff in lib.c:is_valid_number - after recent changes. Thanks to Ozan Yigit. - -December 18, 2020: - Fix problems converting inf and NaN values in lib.c:is_valid_number. - Enhance number to string conversion to do the right thing for - NaN and inf values. Things are now pretty much the same as in - gawk. (Found a gawk bug while we're at it.) Added a torture - test for these values. Thanks to Arnold Robbins. Allows closing - of PR #101. - -December 15, 2020: - Merge PR #99, which gets the right header for strcasecmp. - Thanks to GitHub user michaelforney. - -December 8, 2020: - Merge PR #98: Disallow hex data. Allow only +nan, -nan, - +inf, -inf (case independent) to give NaN and infinity values. - Improve things so that string to double conversion is only - done once, yielding something of a speedup. This obviate - PR #95. Thanks to Arnold Robbins. - -December 3, 2020: - Fix to argument parsing to avoid printing spurious newlines. - Thanks to Todd Miller. Merges PR #97. - -October 13, 2020: - Add casts before all the calls to malloc/calloc/realloc in order - to get it to compile with g++. Thanks to Arnold Robbins. - -August 16, 2020: - Additional fixes for DJGPP. Thanks to Eli Zaretskii for - the testing. - -August 7, 2020: - Merge PR #93, which adds casts to (void*) for debug prints - using the %p format specifier. Thanks to GitHub user YongHaoWu - ("Chris") for the fixes. - -August 4, 2020: - In run.c, use non-restartable multibyte routines to attain - portability to DJGPP. Should fix Issue 92. Thanks to Albert Wik - for the report and to Todd Miller for the suggested fix. - -July 30, 2020: - Merge PRs 88-91 which fix small bugs. Thanks to Todd Miller and - Tim van der Molen for the fixes. - - In order to make life easier, we move exclusively to bison - as the parser generator. - -July 2, 2020: - Merge PRs 85 and 86 which fix regressions. Thanks to - Tim van der Molen for the fixes. - -June 25, 2020: - Merge PRs 82 and 84. The latter fixes issue #83. Thanks to - Todd Miller and awkfan77. - -June 12, 2020: - Clear errno before calling errcheck to avoid any spurious errors - left over from previous calls that may have set it. Thanks to - Todd Miller for the fix, from PR #80. - - Fix Issue #78 by allowing \r to follow floating point numbers in - lib.c:is_number. Thanks to GitHub user ajcarr for the report - and to Arnold Robbins for the fix. - -June 5, 2020: - In fldbld(), make sure that inputFS is set before trying to - use it. Thanks to Steffen Nurpmeso - for the report. - -May 5, 2020: - Fix checks for compilers that can handle noreturn. Thanks to - GitHub user enh-google for pointing it out. Closes Issue #79. - -April 16, 2020: - Handle old compilers that don't support C11 (for noreturn). - Thanks to Arnold Robbins. - -April 5, 2020: - Use and noreturn instead of GCC attributes. - Thanks to GitHub user awkfan77. Closes PR #77. - -February 28, 2020: - More cleanups from Christos Zoulas: notably backslash continuation - inside strings removes the newline and a fix for RS = "^a". - Fix for address sanitizer-found problem. Thanks to GitHub user - enh-google. - -February 19, 2020: - More small cleanups from Christos Zoulas. - -February 18, 2020: - Additional cleanups from Christos Zoulas. It's no longer necessary - to use the -y flag to bison. - -February 6, 2020: - Additional small cleanups from Christos Zoulas. awk is now - a little more robust about reporting I/O errors upon exit. - -January 31, 2020: - Merge PR #70, which avoids use of variable length arrays. Thanks - to GitHub user michaelforney. Fix issue #60 ({0} in interval - expressions doesn't work). Also get all tests working again. - Thanks to Arnold Robbins. - -January 24, 2020: - A number of small cleanups from Christos Zoulas. Add the close - on exec flag to files/pipes opened for redirection; courtesy of - Arnold Robbins. - -January 19, 2020: - If POSIXLY_CORRECT is set in the environment, then sub and gsub - use POSIX rules for multiple backslashes. This fixes Issue #66, - while maintaining backwards compatibility. - -January 9, 2020: - Input/output errors on closing files are now fatal instead of - mere warnings. Thanks to Martijn Dekker . - -January 5, 2020: - Fix a bug in the concatentation of two string constants into - one done in the grammar. Fixes GitHub issue #61. Thanks - to GitHub user awkfan77 for pointing out the direction for - the fix. New test T.concat added to the test suite. - Fix a few memory leaks reported by valgrind, as well. - -December 27, 2019: - Fix a bug whereby a{0,3} could match four a's. Thanks to - "Anonymous AWK fan" for the report. - -December 11, 2019: - Further printf-related fixes for 32 bit systems. - Thanks again to Christos Zoulas. - -December 8, 2019: - Fix the return value of sprintf("%d") on 32 bit systems. - Thanks to Jim Lowe for the report and to Christos Zoulas - for the fix. - -November 10, 2019: - Convert a number of Boolean integer variables into - actual bools. Convert compile_time variable into an - enum and simplify some of the related code. Thanks - to Arnold Robbins. - -November 8, 2019: - Fix from Ori Bernstein to get UTF-8 characters instead of - bytes when FS = "". This is currently the only bit of - the One True Awk that understands multibyte characters. - From Arnold Robbins, apply some cleanups in the test suite. - -October 25, 2019: - More fixes and cleanups from NetBSD, courtesy of Christos - Zoulas. Merges PRs 54 and 55. - -October 24, 2019: - Import second round of code cleanups from NetBSD. Much thanks - to Christos Zoulas (GitHub user zoulasc). Merges PR 53. - Add an optimization for string concatenation, also from - Christos. - -October 17, 2019: - Import code cleanups from NetBSD. Much thanks to Christos - Zoulas (GitHub user zoulasc). Merges PR 51. - -October 6, 2019: - Import code from NetBSD awk that implements RS as a regular - expression. - -September 10, 2019: - Fixes for various array / memory overruns found via gcc's - -fsanitize=unknown. Thanks to Alexander Richardson (GitHub - user arichardson). Merges PRs 47 and 48. - -July 28, 2019: - Import grammar optimization from NetBSD: Two string constants - concatenated together get turned into a single string. - -July 26, 2019: - Support POSIX-specified C-style escape sequences "\a" (alarm) - and "\v" (vertical tab) in command line arguments and regular - expressions, further to the support for them in strings added on - Apr 9, 1989. These now no longer match as literal "a" and "v" - characters (as they don't on other awk implementations). - Thanks to Martijn Dekker. - -July 17, 2019: - Pull in a number of code cleanups and minor fixes from - Warner Losh's bsd-ota branch. The only user visible change - is the use of random(3) as the random number generator. - Thanks to Warner Losh for collecting all these fixes in - one easy place to get them from. - -July 16, 2019: - Fix field splitting to use FS value as of the time a record - was read or assigned to. Thanks to GitHub user Cody Mello (melloc) - for the fix. (Merged from his branch, via PR #42.) Updated - testdir/T.split per said PR as well. - -June 24, 2019: - Extract awktest.tar into testdir directory. Add some very - simple mechanics to the makefile for running the tests and - for cleaning up. No changes to awk itself. - -June 17, 2019: - Disallow deleting SYMTAB and its elements, which creates - use-after-free bugs. Thanks to GitHub user Cody Mello (melloc) - for the fix. (Merged from PR #43.) - -June 5, 2019: - Allow unmatched right parenthesis in a regular expression to - be treated literally. Fixes Issue #40. Thanks to GitHub user - Warner Losh (bsdimp) for the report. Thanks to Arnold Robbins - for the fix. - -May 29,2019: - Fix check for command line arguments to no longer require that - first character after '=' not be another '='. Reverts change of - August 11, 1989. Thanks to GitHub user Jamie Landeg Jones for - pointing out the issue; from Issue #38. - -Apr 7, 2019: - Update awktest.tar(p.50) to use modern options to sort. Needed - for Android development. Thanks to GitHub user mohd-akram (Mohamed - Akram). From Issue #33. - -Mar 12, 2019: - Added very simplistic support for cross-compiling in the - makefile. We are NOT going to go in the direction of the - autotools, though. Thanks to GitHub user nee-san for - the basic change. (Merged from PR #34.) - -Mar 5, 2019: - Added support for POSIX-standard interval expressions (a.k.a. - bounds, a.k.a. repetition expressions) in regular expressions, - backported (via NetBSD) from Apple awk-24 (20070501). - Thanks to Martijn Dekker for the port. - (Merged from PR #30.) - -Mar 3, 2019: - Merge PRs as follows: - #12: Avoid undefined behaviour when using ctype(3) functions in - relex(). Thanks to GitHub user iamleot. - #31: Make getline handle numeric strings, and update FIXES. Thanks - to GitHub user arnoldrobbins. - #32: maketab: support build systems with read-only source. Thanks - to GitHub user enh. - -Jan 25, 2019: - Make getline handle numeric strings properly in all cases. - (Thanks, Arnold.) - -Jan 21, 2019: - Merged a number of small fixes from GitHub pull requests. - Thanks to GitHub users Arnold Robbins (arnoldrobbins), - Cody Mello (melloc) and Christoph Junghans (junghans). - PR numbers: 13-21, 23, 24, 27. - -Oct 25, 2018: - Added test in maketab.c to prevent generating a proctab entry - for YYSTYPE_IS_DEFINED. It was harmless but some gcc settings - generated a warning message. Thanks to Nan Xiao for report. - -Aug 27, 2018: - Disallow '$' in printf formats; arguments evaluated in order - and printed in order. - - Added some casts to silence warnings on debugging printfs. - (Thanks, Arnold.) - -Aug 23, 2018: - A long list of fixes courtesy of Arnold Robbins, - to whom profound thanks. - - 1. ofs-rebuild: OFS value used to rebuild the record was incorrect. - Fixed August 19, 2014. Revised fix August 2018. - - 2. system-status: Instead of a floating-point division by 256, use - the wait(2) macros to create a reasonable exit status. - Fixed March 12, 2016. - - 3. space: Use provided xisblank() function instead of ispace() for - matching [[:blank:]]. - - 4. a-format: Add POSIX standard %a and %A to supported formats. Check - at runtime that this format is available. - - 5. decr-NF: Decrementing NF did not change $0. This is a decades-old - bug. There are interactions with the old and new value of OFS as well. - Most of the fix came from the NetBSD awk. - - 6. string-conv: String conversions of scalars were sticky. Once a - conversion to string happened, even with OFMT, that value was used until - a new numeric value was assigned, even if OFMT differed from CONVFMT, - and also if CONVFMT changed. - - 7. unary-plus: Unary plus on a string constant returned the string. - Instead, it should convert the value to numeric and give that value. - - Also added Arnold's tests for these to awktest.tar as T.arnold. - -Aug 15, 2018: - fixed mangled awktest.tar (thanks, Arnold), posted all - current (very minor) fixes to github / onetrueawk - -Jun 7, 2018: - (yes, a long layoff) - Updated some broken tests (beebe.tar, T.lilly) - [thanks to Arnold Robbins] - -Mar 26, 2015: - buffer overflow in error reporting; thanks to tobias ulmer - and john-mark gurney for spotting it and the fix. - -Feb 4, 2013: - cleaned up a handful of tests that didn't seem to actually - test for correct behavior: T.latin1, T.gawk. - -Jan 5, 2013: - added ,NULL initializer to static Cells in run.c; not really - needed but cleaner. Thanks to Michael Bombardieri. - -Dec 20, 2012: - fiddled makefile to get correct yacc and bison flags. pick yacc - (linux) or bison (mac) as necessary. - - added __attribute__((__noreturn__)) to a couple of lines in - proto.h, to silence someone's enthusiastic checker. - - fixed obscure call by value bug in split(a[1],a) reported on - 9fans. the management of temporary values is just a mess; i - took a shortcut by making an extra string copy. thanks - to paul patience and arnold robbins for passing it on and for - proposed patches. - - tiny fiddle in setfval to eliminate -0 results in T.expr, which - has irritated me for 20+ years. - -Aug 10, 2011: - another fix to avoid core dump with delete(ARGV); again, many thanks - to ruslan ermilov. - -Aug 7, 2011: - split(s, a, //) now behaves the same as split(s, a, "") - -Jun 12, 2011: - /pat/, \n /pat/ {...} is now legal, though bad style to use. - - added checks to new -v code that permits -vnospace; thanks to - ruslan ermilov for spotting this and providing the patch. - - removed fixed limit on number of open files; thanks to aleksey - cheusov and christos zoulos. - - fixed day 1 bug that resurrected deleted elements of ARGV when - used as filenames (in lib.c). - - minor type fiddles to make gcc -Wall -pedantic happier (but not - totally so); turned on -fno-strict-aliasing in makefile. - -May 6, 2011: - added #ifdef for isblank. - now allows -ffoo as well as -f foo arguments. - (thanks, ruslan) - -May 1, 2011: - after advice from todd miller, kevin lo, ruslan ermilov, - and arnold robbins, changed srand() to return the previous - seed (which is 1 on the first call of srand). the seed is - an Awkfloat internally though converted to unsigned int to - pass to the library srand(). thanks, everyone. - - fixed a subtle (and i hope low-probability) overflow error - in fldbld, by adding space for one extra \0. thanks to - robert bassett for spotting this one and providing a fix. - - removed the files related to compilation on windows. i no - longer have anything like a current windows environment, so - i can't test any of it. - -May 23, 2010: - fixed long-standing overflow bug in run.c; many thanks to - nelson beebe for spotting it and providing the fix. - - fixed bug that didn't parse -vd=1 properly; thanks to santiago - vila for spotting it. - -Feb 8, 2010: - i give up. replaced isblank with isspace in b.c; there are - no consistent header files. - -Nov 26, 2009: - fixed a long-standing issue with when FS takes effect. a - change to FS is now noticed immediately for subsequent splits. - - changed the name getline() to awkgetline() to avoid yet another - name conflict somewhere. - -Feb 11, 2009: - temporarily for now defined HAS_ISBLANK, since that seems to - be the best way through the thicket. isblank arrived in C99, - but seems to be arriving at different systems at different - times. - -Oct 8, 2008: - fixed typo in b.c that set tmpvec wrongly. no one had ever - run into the problem, apparently. thanks to alistair crooks. - -Oct 23, 2007: - minor fix in lib.c: increase inputFS to 100, change malloc - for fields to n+1. - - fixed memory fault caused by out of order test in setsval. - - thanks to david o'brien, freebsd, for both fixes. - -May 1, 2007: - fiddle in makefile to fix for BSD make; thanks to igor sobrado. - -Mar 31, 2007: - fixed some null pointer refs calling adjbuf. - -Feb 21, 2007: - fixed a bug in matching the null RE in sub and gsub. thanks to al aho - who actually did the fix (in b.c), and to wolfgang seeberg for finding - it and providing a very compact test case. - - fixed quotation in b.c; thanks to Hal Pratt and the Princeton Dante - Project. - - removed some no-effect asserts in run.c. - - fiddled maketab.c to not complain about bison-generated values. - - removed the obsolete -V argument; fixed --version to print the - version and exit. - - fixed wording and an outright error in the usage message; thanks to igor - sobrado and jason mcintyre. - - fixed a bug in -d that caused core dump if no program followed. - -Jan 1, 2007: - dropped mac.code from makefile; there are few non-MacOSX - mac's these days. - -Jan 17, 2006: - system() not flagged as unsafe in the unadvertised -safe option. - found it while enhancing tests before shipping the ;login: article. - practice what you preach. - - removed the 9-years-obsolete -mr and -mf flags. - - added -version and --version options. - - core dump on linux with BEGIN {nextfile}, now fixed. - - removed some #ifdef's in run.c and lex.c that appear to no - longer be necessary. - -Apr 24, 2005: - modified lib.c so that values of $0 et al are preserved in the END - block, apparently as required by posix. thanks to havard eidnes - for the report and code. - -Jan 14, 2005: - fixed infinite loop in parsing, originally found by brian tsang. - thanks to arnold robbins for a suggestion that started me - rethinking it. - -Dec 31, 2004: - prevent overflow of -f array in main, head off potential error in - call of SYNTAX(), test malloc return in lib.c, all with thanks to - todd miller. - -Dec 22, 2004: - cranked up size of NCHARS; coverity thinks it can be overrun with - smaller size, and i think that's right. added some assertions to b.c - to catch places where it might overrun. the RE code is still fragile. - -Dec 5, 2004: - fixed a couple of overflow problems with ridiculous field numbers: - e.g., print $(2^32-1). thanks to ruslan ermilov, giorgos keramidas - and david o'brien at freebsd.org for patches. this really should - be re-done from scratch. - -Nov 21, 2004: - fixed another 25-year-old RE bug, in split. it's another failure - to (re-)initialize. thanks to steve fisher for spotting this and - providing a good test case. - -Nov 22, 2003: - fixed a bug in regular expressions that dates (so help me) from 1977; - it's been there from the beginning. an anchored longest match that - was longer than the number of states triggered a failure to initialize - the machine properly. many thanks to moinak ghosh for not only finding - this one but for providing a fix, in some of the most mysterious - code known to man. - - fixed a storage leak in call() that appears to have been there since - 1983 or so -- a function without an explicit return that assigns a - string to a parameter leaked a Cell. thanks to moinak ghosh for - spotting this very subtle one. - -Jul 31, 2003: - fixed, thanks to andrey chernov and ruslan ermilov, a bug in lex.c - that mis-handled the character 255 in input. (it was being compared - to EOF with a signed comparison.) - -Jul 29, 2003: - fixed (i think) the long-standing botch that included the beginning of - line state ^ for RE's in the set of valid characters; this led to a - variety of odd problems, including failure to properly match certain - regular expressions in non-US locales. thanks to ruslan for keeping - at this one. - -Jul 28, 2003: - n-th try at getting internationalization right, with thanks to volker - kiefel, arnold robbins and ruslan ermilov for advice, though they - should not be blamed for the outcome. according to posix, "." is the - radix character in programs and command line arguments regardless of - the locale; otherwise, the locale should prevail for input and output - of numbers. so it's intended to work that way. - - i have rescinded the attempt to use strcoll in expanding shorthands in - regular expressions (cclenter). its properties are much too - surprising; for example [a-c] matches aAbBc in locale en_US but abBcC - in locale fr_CA. i can see how this might arise by implementation - but i cannot explain it to a human user. (this behavior can be seen - in gawk as well; we're leaning on the same library.) - - the issue appears to be that strcoll is meant for sorting, where - merging upper and lower case may make sense (though note that unix - sort does not do this by default either). it is not appropriate - for regular expressions, where the goal is to match specific - patterns of characters. in any case, the notations [:lower:], etc., - are available in awk, and they are more likely to work correctly in - most locales. - - a moratorium is hereby declared on internationalization changes. - i apologize to friends and colleagues in other parts of the world. - i would truly like to get this "right", but i don't know what - that is, and i do not want to keep making changes until it's clear. - -Jul 4, 2003: - fixed bug that permitted non-terminated RE, as in "awk /x". - -Jun 1, 2003: - subtle change to split: if source is empty, number of elems - is always 0 and the array is not set. - -Mar 21, 2003: - added some parens to isblank, in another attempt to make things - internationally portable. - -Mar 14, 2003: - the internationalization changes, somewhat modified, are now - reinstated. in theory awk will now do character comparisons - and case conversions in national language, but "." will always - be the decimal point separator on input and output regardless - of national language. isblank(){} has an #ifndef. - - this no longer compiles on windows: LC_MESSAGES isn't defined - in vc6++. - - fixed subtle behavior in field and record splitting: if FS is - a single character and RS is not empty, \n is NOT a separator. - this tortuous reading is found in the awk book; behavior now - matches gawk and mawk. - -Dec 13, 2002: - for the moment, the internationalization changes of nov 29 are - rolled back -- programs like x = 1.2 don't work in some locales, - because the parser is expecting x = 1,2. until i understand this - better, this will have to wait. - -Nov 29, 2002: - modified b.c (with tiny changes in main and run) to support - locales, using strcoll and iswhatever tests for posix character - classes. thanks to ruslan ermilov (ru@freebsd.org) for code. - the function isblank doesn't seem to have propagated to any - header file near me, so it's there explicitly. not properly - tested on non-ascii character sets by me. - -Jun 28, 2002: - modified run/format() and tran/getsval() to do a slightly better - job on using OFMT for output from print and CONVFMT for other - number->string conversions, as promised by posix and done by - gawk and mawk. there are still places where it doesn't work - right if CONVFMT is changed; by then the STR attribute of the - variable has been irrevocably set. thanks to arnold robbins for - code and examples. - - fixed subtle bug in format that could get core dump. thanks to - Jaromir Dolecek for finding and fixing. - minor cleanup in run.c / format() at the same time. - - added some tests for null pointers to debugging printf's, which - were never intended for external consumption. thanks to dave - kerns (dkerns@lucent.com) for pointing this out. - - GNU compatibility: an empty regexp matches anything (thanks to - dag-erling smorgrav, des@ofug.org). subject to reversion if - this does more harm than good. - - pervasive small changes to make things more const-correct, as - reported by gcc's -Wwrite-strings. as it says in the gcc manual, - this may be more nuisance than useful. provoked by a suggestion - and code from arnaud desitter, arnaud@nimbus.geog.ox.ac.uk - - minor documentation changes to note that this now compiles out - of the box on Mac OS X. - -Feb 10, 2002: - changed types in posix chars structure to quiet solaris cc. - -Jan 1, 2002: - fflush() or fflush("") flushes all files and pipes. - - length(arrayname) returns number of elements; thanks to - arnold robbins for suggestion. - - added a makefile.win to make it easier to build on windows. - based on dan allen's buildwin.bat. - -Nov 16, 2001: - added support for posix character class names like [:digit:], - which are not exactly shorter than [0-9] and perhaps no more - portable. thanks to dag-erling smorgrav for code. - -Feb 16, 2001: - removed -m option; no longer needed, and it was actually - broken (noted thanks to volker kiefel). - -Feb 10, 2001: - fixed an appalling bug in gettok: any sequence of digits, +,-, E, e, - and period was accepted as a valid number if it started with a period. - this would never have happened with the lex version. - - other 1-character botches, now fixed, include a bare $ and a - bare " at the end of the input. - -Feb 7, 2001: - more (const char *) casts in b.c and tran.c to silence warnings. - -Nov 15, 2000: - fixed a bug introduced in august 1997 that caused expressions - like $f[1] to be syntax errors. thanks to arnold robbins for - noticing this and providing a fix. - -Oct 30, 2000: - fixed some nextfile bugs: not handling all cases. thanks to - arnold robbins for pointing this out. new regressions added. - - close() is now a function. it returns whatever the library - fclose returns, and -1 for closing a file or pipe that wasn't - opened. - -Sep 24, 2000: - permit \n explicitly in character classes; won't work right - if comes in as "[\n]" but ok as /[\n]/, because of multiple - processing of \'s. thanks to arnold robbins. - -July 5, 2000: - minor fiddles in tran.c to keep compilers happy about uschar. - thanks to norman wilson. - -May 25, 2000: - yet another attempt at making 8-bit input work, with another - band-aid in b.c (member()), and some (uschar) casts to head - off potential errors in subscripts (like isdigit). also - changed HAT to NCHARS-2. thanks again to santiago vila. - - changed maketab.c to ignore apparently out of range definitions - instead of halting; new freeBSD generates one. thanks to - jon snader for pointing out the problem. - -May 2, 2000: - fixed an 8-bit problem in b.c by making several char*'s into - unsigned char*'s. not clear i have them all yet. thanks to - Santiago Vila for the bug report. - -Apr 21, 2000: - finally found and fixed a memory leak in function call; it's - been there since functions were added ~1983. thanks to - jon bentley for the test case that found it. - - added test in envinit to catch environment "variables" with - names beginning with '='; thanks to Berend Hasselman. - -Jul 28, 1999: - added test in defn() to catch function foo(foo), which - otherwise recurses until core dump. thanks to arnold - robbins for noticing this. - -Jun 20, 1999: - added *bp in gettok in lex.c; appears possible to exit function - without terminating the string. thanks to russ cox. - -Jun 2, 1999: - added function stdinit() to run to initialize files[] array, - in case stdin, etc., are not constants; some compilers care. - -May 10, 1999: - replaced the ERROR ... FATAL, etc., macros with functions - based on vprintf, to avoid problems caused by overrunning - fixed-size errbuf array. thanks to ralph corderoy for the - impetus, and for pointing out a string termination bug in - qstring as well. - -Apr 21, 1999: - fixed bug that caused occasional core dumps with commandline - variable with value ending in \. (thanks to nelson beebe for - the test case.) - -Apr 16, 1999: - with code kindly provided by Bruce Lilly, awk now parses - /=/ and similar constructs more sensibly in more places. - Bruce also provided some helpful test cases. - -Apr 5, 1999: - changed true/false to True/False in run.c to make it - easier to compile with C++. Added some casts on malloc - and realloc to be honest about casts; ditto. changed - ltype int to long in struct rrow to reduce some 64-bit - complaints; other changes scattered throughout for the - same purpose. thanks to Nelson Beebe for these portability - improvements. - - removed some horrible pointer-int casting in b.c and elsewhere - by adding ptoi and itonp to localize the casts, which are - all benign. fixed one incipient bug that showed up on sgi - in 64-bit mode. - - reset lineno for new source file; include filename in error - message. also fixed line number error in continuation lines. - (thanks to Nelson Beebe for both of these.) - -Mar 24, 1999: - Nelson Beebe notes that irix 5.3 yacc dies with a bogus - error; use a newer version or switch to bison, since sgi - is unlikely to fix it. - -Mar 5, 1999: - changed isnumber to is_number to avoid the problem caused by - versions of ctype.h that include the name isnumber. - - distribution now includes a script for building on a Mac, - thanks to Dan Allen. - -Feb 20, 1999: - fixed memory leaks in run.c (call) and tran.c (setfval). - thanks to Stephen Nutt for finding these and providing the fixes. - -Jan 13, 1999: - replaced srand argument by (unsigned int) in run.c; - avoids problem on Mac and potentially on Unix & Windows. - thanks to Dan Allen. - - added a few (int) casts to silence useless compiler warnings. - e.g., errorflag= in run.c jump(). - - added proctab.c to the bundle outout; one less thing - to have to compile out of the box. - - added calls to _popen and _pclose to the win95 stub for - pipes (thanks to Steve Adams for this helpful suggestion). - seems to work, though properties are not well understood - by me, and it appears that under some circumstances the - pipe output is truncated. Be careful. - -Oct 19, 1998: - fixed a couple of bugs in getrec: could fail to update $0 - after a getline var; because inputFS wasn't initialized, - could split $0 on every character, a misleading diversion. - - fixed caching bug in makedfa: LRU was actually removing - least often used. - - thanks to ross ridge for finding these, and for providing - great bug reports. - -May 12, 1998: - fixed potential bug in readrec: might fail to update record - pointer after growing. thanks to dan levy for spotting this - and suggesting the fix. - -Mar 12, 1998: - added -V to print version number and die. - -[notify dave kerns, dkerns@dacsoup.ih.lucent.com] - -Feb 11, 1998: - subtle silent bug in lex.c: if the program ended with a number - longer than 1 digit, part of the input would be pushed back and - parsed again because token buffer wasn't terminated right. - example: awk 'length($0) > 10'. blush. at least i found it - myself. - -Aug 31, 1997: - s/adelete/awkdelete/: SGI uses this in malloc.h. - thanks to nelson beebe for pointing this one out. - -Aug 21, 1997: - fixed some bugs in sub and gsub when replacement includes \\. - this is a dark, horrible corner, but at least now i believe that - the behavior is the same as gawk and the intended posix standard. - thanks to arnold robbins for advice here. - -Aug 9, 1997: - somewhat regretfully, replaced the ancient lex-based lexical - analyzer with one written in C. it's longer, generates less code, - and more portable; the old one depended too much on mysterious - properties of lex that were not preserved in other environments. - in theory these recognize the same language. - - now using strtod to test whether a string is a number, instead of - the convoluted original function. should be more portable and - reliable if strtod is implemented right. - - removed now-pointless optimization in makefile that tries to avoid - recompilation when awkgram.y is changed but symbols are not. - - removed most fixed-size arrays, though a handful remain, some - of which are unchecked. you have been warned. - -Aug 4, 1997: - with some trepidation, replaced the ancient code that managed - fields and $0 in fixed-size arrays with arrays that grow on - demand. there is still some tension between trying to make this - run fast and making it clean; not sure it's right yet. - - the ill-conceived -mr and -mf arguments are now useful only - for debugging. previous dynamic string code removed. - - numerous other minor cleanups along the way. - -Jul 30, 1997: - using code provided by dan levy (to whom profuse thanks), replaced - fixed-size arrays and awkward kludges by a fairly uniform mechanism - to grow arrays as needed for printf, sub, gsub, etc. - -Jul 23, 1997: - falling off the end of a function returns "" and 0, not 0. - thanks to arnold robbins. - -Jun 17, 1997: - replaced several fixed-size arrays by dynamically-created ones - in run.c; added overflow tests to some previously unchecked cases. - getline, toupper, tolower. - - getline code is still broken in that recursive calls may wind - up using the same space. [fixed later] - - increased RECSIZE to 8192 to push problems further over the horizon. - - added \r to \n as input line separator for programs, not data. - damn CRLFs. - - modified format() to permit explicit printf("%c", 0) to include - a null byte in output. thanks to ken stailey for the fix. - - added a "-safe" argument that disables file output (print >, - print >>), process creation (cmd|getline, print |, system), and - access to the environment (ENVIRON). this is a first approximation - to a "safe" version of awk, but don't rely on it too much. thanks - to joan feigenbaum and matt blaze for the inspiration long ago. - -Jul 8, 1996: - fixed long-standing bug in sub, gsub(/a/, "\\\\&"); thanks to - ralph corderoy. - -Jun 29, 1996: - fixed awful bug in new field splitting; didn't get all the places - where input was done. - -Jun 28, 1996: - changed field-splitting to conform to posix definition: fields are - split using the value of FS at the time of input; it used to be - the value when the field or NF was first referred to, a much less - predictable definition. thanks to arnold robbins for encouragement - to do the right thing. - -May 28, 1996: - fixed appalling but apparently unimportant bug in parsing octal - numbers in reg exprs. - - explicit hex in reg exprs now limited to 2 chars: \xa, \xaa. - -May 27, 1996: - cleaned up some declarations so gcc -Wall is now almost silent. - - makefile now includes backup copies of ytab.c and lexyy.c in case - one makes before looking; it also avoids recreating lexyy.c unless - really needed. - - s/aprintf/awkprint, s/asprintf/awksprintf/ to avoid some name clashes - with unwisely-written header files. - - thanks to jeffrey friedl for several of these. - -May 26, 1996: - an attempt to rationalize the (unsigned) char issue. almost all - instances of unsigned char have been removed; the handful of places - in b.c where chars are used as table indices have been hand-crafted. - added some latin-1 tests to the regression, but i'm not confident; - none of my compilers seem to care much. thanks to nelson beebe for - pointing out some others that do care. - -May 2, 1996: - removed all register declarations. - - enhanced split(), as in gawk, etc: split(s, a, "") splits s into - a[1]...a[length(s)] with each character a single element. - - made the same changes for field-splitting if FS is "". - - added nextfile, as in gawk: causes immediate advance to next - input file. (thanks to arnold robbins for inspiration and code). - - small fixes to regexpr code: can now handle []], [[], and - variants; [] is now a syntax error, rather than matching - everything; [z-a] is now empty, not z. far from complete - or correct, however. (thanks to jeffrey friedl for pointing out - some awful behaviors.) - -Apr 29, 1996: - replaced uchar by uschar everywhere; apparently some compilers - usurp this name and this causes conflicts. - - fixed call to time in run.c (bltin); arg is time_t *. - - replaced horrible pointer/long punning in b.c by a legitimate - union. should be safer on 64-bit machines and cleaner everywhere. - (thanks to nelson beebe for pointing out some of these problems.) - - replaced nested comments by #if 0...#endif in run.c, lib.c. - - removed getsval, setsval, execute macros from run.c and lib.c. - machines are 100x faster than they were when these macros were - first used. - - revised filenames: awk.g.y => awkgram.y, awk.lx.l => awklex.l, - y.tab.[ch] => ytab.[ch], lex.yy.c => lexyy.c, all in the aid of - portability to nameless systems. - - "make bundle" now includes yacc and lex output files for recipients - who don't have yacc or lex. - -Aug 15, 1995: - initialized Cells in setsymtab more carefully; some fields - were not set. (thanks to purify, all of whose complaints i - think i now understand.) - - fixed at least one error in gsub that looked at -1-th element - of an array when substituting for a null match (e.g., $). - - delete arrayname is now legal; it clears the elements but leaves - the array, which may not be the right behavior. - - modified makefile: my current make can't cope with the test used - to avoid unnecessary yacc invocations. - -Jul 17, 1995: - added dynamically growing strings to awk.lx.l and b.c - to permit regular expressions to be much bigger. - the state arrays can still overflow. - -Aug 24, 1994: - detect duplicate arguments in function definitions (mdm). - -May 11, 1994: - trivial fix to printf to limit string size in sub(). - -Apr 22, 1994: - fixed yet another subtle self-assignment problem: - $1 = $2; $1 = $1 clobbered $1. - - Regression tests now use private echo, to avoid quoting problems. - -Feb 2, 1994: - changed error() to print line number as %d, not %g. - -Jul 23, 1993: - cosmetic changes: increased sizes of some arrays, - reworded some error messages. - - added CONVFMT as in posix (just replaced OFMT in getsval) - - FILENAME is now "" until the first thing that causes a file - to be opened. - -Nov 28, 1992: - deleted yyunput and yyoutput from proto.h; - different versions of lex give these different declarations. - -May 31, 1992: - added -mr N and -mf N options: more record and fields. - these really ought to adjust automatically. - - cleaned up some error messages; "out of space" now means - malloc returned NULL in all cases. - - changed rehash so that if it runs out, it just returns; - things will continue to run slow, but maybe a bit longer. - -Apr 24, 1992: - remove redundant close of stdin when using -f -. - - got rid of core dump with -d; awk -d just prints date. - -Apr 12, 1992: - added explicit check for /dev/std(in,out,err) in redirection. - unlike gawk, no /dev/fd/n yet. - - added (file/pipe) builtin. hard to test satisfactorily. - not posix. - -Feb 20, 1992: - recompile after abortive changes; should be unchanged. - -Dec 2, 1991: - die-casting time: converted to ansi C, installed that. - -Nov 30, 1991: - fixed storage leak in freefa, failing to recover [N]CCL. - thanks to Bill Jones (jones@cs.usask.ca) - -Nov 19, 1991: - use RAND_MAX instead of literal in builtin(). - -Nov 12, 1991: - cranked up some fixed-size arrays in b.c, and added a test for - overflow in penter. thanks to mark larsen. - -Sep 24, 1991: - increased buffer in gsub. a very crude fix to a general problem. - and again on Sep 26. - -Aug 18, 1991: - enforce variable name syntax for commandline variables: has to - start with letter or _. - -Jul 27, 1991: - allow newline after ; in for statements. - -Jul 21, 1991: - fixed so that in self-assignment like $1=$1, side effects - like recomputing $0 take place. (this is getting subtle.) - -Jun 30, 1991: - better test for detecting too-long output record. - -Jun 2, 1991: - better defense against very long printf strings. - made break and continue illegal outside of loops. - -May 13, 1991: - removed extra arg on gettemp, tempfree. minor error message rewording. - -May 6, 1991: - fixed silly bug in hex parsing in hexstr(). - removed an apparently unnecessary test in isnumber(). - warn about weird printf conversions. - fixed unchecked array overwrite in relex(). - - changed for (i in array) to access elements in sorted order. - then unchanged it -- it really does run slower in too many cases. - left the code in place, commented out. - -Feb 10, 1991: - check error status on all writes, to avoid banging on full disks. - -Jan 28, 1991: - awk -f - reads the program from stdin. - -Jan 11, 1991: - failed to set numeric state on $0 in cmd|getline context in run.c. - -Nov 2, 1990: - fixed sleazy test for integrality in getsval; use modf. - -Oct 29, 1990: - fixed sleazy buggy code in lib.c that looked (incorrectly) for - too long input lines. - -Oct 14, 1990: - fixed the bug on p. 198 in which it couldn't deduce that an - argument was an array in some contexts. replaced the error - message in intest() by code that damn well makes it an array. - -Oct 8, 1990: - fixed horrible bug: types and values were not preserved in - some kinds of self-assignment. (in assign().) - -Aug 24, 1990: - changed NCHARS to 256 to handle 8-bit characters in strings - presented to match(), etc. - -Jun 26, 1990: - changed struct rrow (awk.h) to use long instead of int for lval, - since cfoll() stores a pointer in it. now works better when int's - are smaller than pointers! - -May 6, 1990: - AVA fixed the grammar so that ! is uniformly of the same precedence as - unary + and -. This renders illegal some constructs like !x=y, which - now has to be parenthesized as !(x=y), and makes others work properly: - !x+y is (!x)+y, and x!y is x !y, not two pattern-action statements. - (These problems were pointed out by Bob Lenk of Posix.) - - Added \x to regular expressions (already in strings). - Limited octal to octal digits; \8 and \9 are not octal. - Centralized the code for parsing escapes in regular expressions. - Added a bunch of tests to T.re and T.sub to verify some of this. - -Feb 9, 1990: - fixed null pointer dereference bug in main.c: -F[nothing]. sigh. - - restored srand behavior: it returns the current seed. - -Jan 18, 1990: - srand now returns previous seed value (0 to start). - -Jan 5, 1990: - fix potential problem in tran.c -- something was freed, - then used in freesymtab. - -Oct 18, 1989: - another try to get the max number of open files set with - relatively machine-independent code. - - small fix to input() in case of multiple reads after EOF. - -Oct 11, 1989: - FILENAME is now defined in the BEGIN block -- too many old - programs broke. - - "-" means stdin in getline as well as on the commandline. - - added a bunch of casts to the code to tell the truth about - char * vs. unsigned char *, a right royal pain. added a - setlocale call to the front of main, though probably no one - has it usefully implemented yet. - -Aug 24, 1989: - removed redundant relational tests against nullnode if parse - tree already had a relational at that point. - -Aug 11, 1989: - fixed bug: commandline variable assignment has to look like - var=something. (consider the man page for =, in file =.1) - - changed number of arguments to functions to static arrays - to avoid repeated malloc calls. - -Aug 2, 1989: - restored -F (space) separator - -Jul 30, 1989: - added -v x=1 y=2 ... for immediate commandline variable assignment; - done before the BEGIN block for sure. they have to precede the - program if the program is on the commandline. - Modified Aug 2 to require a separate -v for each assignment. - -Jul 10, 1989: - fixed ref-thru-zero bug in environment code in tran.c - -Jun 23, 1989: - add newline to usage message. - -Jun 14, 1989: - added some missing ansi printf conversion letters: %i %X %E %G. - no sensible meaning for h or L, so they may not do what one expects. - - made %* conversions work. - - changed x^y so that if n is a positive integer, it's done - by explicit multiplication, thus achieving maximum accuracy. - (this should be done by pow() but it seems not to be locally.) - done to x ^= y as well. - -Jun 4, 1989: - ENVIRON array contains environment: if shell variable V=thing, - ENVIRON["V"] is "thing" - - multiple -f arguments permitted. error reporting is naive. - (they were permitted before, but only the last was used.) - - fixed a really stupid botch in the debugging macro dprintf - - fixed order of evaluation of commandline assignments to match - what the book claims: an argument of the form x=e is evaluated - at the time it would have been opened if it were a filename (p 63). - this invalidates the suggested answer to ex 4-1 (p 195). - - removed some code that permitted -F (space) fieldseparator, - since it didn't quite work right anyway. (restored aug 2) - -Apr 27, 1989: - Line number now accumulated correctly for comment lines. - -Apr 26, 1989: - Debugging output now includes a version date, - if one compiles it into the source each time. - -Apr 9, 1989: - Changed grammar to prohibit constants as 3rd arg of sub and gsub; - prevents class of overwriting-a-constant errors. (Last one?) - This invalidates the "banana" example on page 43 of the book. - - Added \a ("alert"), \v (vertical tab), \xhhh (hexadecimal), - as in ANSI, for strings. Rescinded the sloppiness that permitted - non-octal digits in \ooo. Warning: not all compilers and libraries - will be able to deal with \x correctly. - -Jan 9, 1989: - Fixed bug that caused tempcell list to contain a duplicate. - The fix is kludgy. - -Dec 17, 1988: - Catches some more commandline errors in main. - Removed redundant decl of modf in run.c (confuses some compilers). - Warning: there's no single declaration of malloc, etc., in awk.h - that seems to satisfy all compilers. - -Dec 7, 1988: - Added a bit of code to error printing to avoid printing nulls. - (Not clear that it actually would.) - -Nov 27, 1988: - With fear and trembling, modified the grammar to permit - multiple pattern-action statements on one line without - an explicit separator. By definition, this capitulation - to the ghost of ancient implementations remains undefined - and thus subject to change without notice or apology. - DO NOT COUNT ON IT. - -Oct 30, 1988: - Fixed bug in call() that failed to recover storage. - - A warning is now generated if there are more arguments - in the call than in the definition (in lieu of fixing - another storage leak). - -Oct 20, 1988: - Fixed %c: if expr is numeric, use numeric value; - otherwise print 1st char of string value. still - doesn't work if the value is 0 -- won't print \0. - - Added a few more checks for running out of malloc. - -Oct 12, 1988: - Fixed bug in call() that freed local arrays twice. - - Fixed to handle deletion of non-existent array right; - complains about attempt to delete non-array element. - -Sep 30, 1988: - Now guarantees to evaluate all arguments of built-in - functions, as in C; the appearance is that arguments - are evaluated before the function is called. Places - affected are sub (gsub was ok), substr, printf, and - all the built-in arithmetic functions in bltin(). - A warning is generated if a bltin() is called with - the wrong number of arguments. - - This requires changing makeprof on p167 of the book. - -Aug 23, 1988: - setting FILENAME in BEGIN caused core dump, apparently - because it was freeing space not allocated by malloc. - -July 24, 1988: - fixed egregious error in toupper/tolower functions. - still subject to rescinding, however. - -July 2, 1988: - flush stdout before opening file or pipe - -July 2, 1988: - performance bug in b.c/cgoto(): not freeing some sets of states. - partial fix only right now, and the number of states increased - to make it less obvious. - -June 1, 1988: - check error status on close - -May 28, 1988: - srand returns seed value it's using. - see 1/18/90 - -May 22, 1988: - Removed limit on depth of function calls. - -May 10, 1988: - Fixed lib.c to permit _ in commandline variable names. - -Mar 25, 1988: - main.c fixed to recognize -- as terminator of command- - line options. Illegal options flagged. - Error reporting slightly cleaned up. - -Dec 2, 1987: - Newer C compilers apply a strict scope rule to extern - declarations within functions. Two extern declarations in - lib.c and tran.c have been moved to obviate this problem. - -Oct xx, 1987: - Reluctantly added toupper and tolower functions. - Subject to rescinding without notice. - -Sep 17, 1987: - Error-message printer had printf(s) instead of - printf("%s",s); got core dumps when the message - included a %. - -Sep 12, 1987: - Very long printf strings caused core dump; - fixed aprintf, asprintf, format to catch them. - Can still get a core dump in printf itself. - - +Dec 24, 2023: + Matchop dereference after free problem fix when the first + argument is a function call. Thanks to Oguz Ismail Uysal. + Fix inconsistent handling of --csv and FS set in the + command line. Thanks to Wilbert van der Poel. + Casting changes to int for is* functions. + +Nov 27, 2023: + Fix exit status of system on MacOS. Update to REGRESS. + Thanks to Arnold Robbins. + Fix inconsistent handling of -F and --csv, and loss of csv + mode when FS is set. + +Nov 24, 2023: + Fix issue #199: gototab improvements to dynamically resize the + table, qsort and bsearch to improve the lookup speed as the + table gets larger for multibyte input. Thanks to Arnold Robbins. + +Nov 23, 2023: + Fix Issue #169, related to escape sequences in strings. + Thanks to Github user rajeevvp. + Fix Issue #147, reported by Github user drawkula, and fixed + by Miguel Pineiro Jr. + +Nov 20, 2023: + Rewrite of fnematch to fix a number of issues, including + extraneous output, out-of-bounds access, number of bytes + to push back after a failed match etc. + Thanks to Miguel Pineiro Jr. + +Nov 15, 2023: + Man page edit, regression test fixes. Thanks to Arnold Robbins + Consolidation of sub and gsub into dosub, removing duplicate + code. Thanks to Miguel Pineiro Jr. + gcc replaced with cc everywhere. + +Oct 30, 2023: + Multiple fixes and a minor code cleanup. + Disabled utf-8 for non-multibyte locales, such as C or POSIX. + Fixed a bad char * cast that causes incorrect results on big-endian + systems. Also fixed an out-of-bounds read for empty CCL. + Fixed a buffer overflow in substr with utf-8 strings. + Many thanks to Todd C Miller. + +Sep 24, 2023: + fnematch and getrune have been overhauled to solve issues around + unicode FS and RS. Also fixed gsub null match issue with unicode. + Big thanks to Arnold Robbins. + +Sep 12, 2023: + Fixed a length error in u8_byte2char that set RSTART to + incorrect (cannot happen) value for EOL match(str, /$/). + + +----------------------------------------------------------------- + +[This entry is a summary, not a precise list of changes.] + + Added --csv option to enable processing of comma-separated + values inputs. When --csv is enabled, fields are separated + by commas, fields may be quoted with " double quotes, fields + may contain embedded newlines. + + If no explicit separator argument is provided, split() uses + the setting of --csv to determine how fields are split. + + Strings may now contain UTF-8 code points (not necessarily + characters). Functions that operate on characters, like + length, substr, index, match, etc., use UTF-8, so the length + of a string of 3 emojis is 3, not 12 as it would be if bytes + were counted. + + Regular expressions are processed as UTF-8. + + Unicode literals can be written as \u followed by one + to eight hexadecimal digits. These may appear in strings and + regular expressions. diff --git a/contrib/one-true-awk/FIXES b/contrib/one-true-awk/FIXES.1e similarity index 95% copy from contrib/one-true-awk/FIXES copy to contrib/one-true-awk/FIXES.1e index 8a2befccd624..8cbd6ac1a097 100644 --- a/contrib/one-true-awk/FIXES +++ b/contrib/one-true-awk/FIXES.1e @@ -1,1375 +1,1429 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ This file lists all bug fixes, changes, etc., made since the AWK book -was sent to the printers in August, 1987. +was sent to the printers in August 1987. + +Sep 06, 2023: + Fix edge case where FS is changed on commandline. Thanks to + Gordon Shephard and Miguel Pineiro Jr. + + Fix regular expression clobbering in the lexer, where lexer does + not make a copy of regexp literals. also makedfa memory leaks have + been plugged. Thanks to Miguel Pineiro Jr. + +Dec 15, 2022: + Force hex escapes in strings to be no more than two characters, + as they already are in regular expressions. This brings internal + consistency, as well as consistency with gawk. Thanks to + Arnold Robbins. + +Sep 12, 2022: + adjbuf minlen error (cannot be 0) in cat, resulting in NULL pbuf. + discovered by todd miller. also use-after-free issue with + tempfree in cat, thanks to Miguel Pineiro Jr and valgrind. + +Aug 30, 2022: + Various leaks and use-after-free issues plugged/fixed. + Thanks to Miguel Pineiro Jr. . + +May 23, 2022: + Memory leak when assigning a string to some of the built-in + variables. allocated string erroneously marked DONTFREE. + Thanks to Miguel Pineiro Jr. . + +Mar 14, 2022: + Historic bug: command-line "name=value" assignment had been + truncating its entry in ARGV. (circa 1989) Thanks to + Miguel Pineiro Jr. . + +Mar 3, 2022: + Fixed file management memory leak that appears to have been + there since the files array was first initialized with stdin, + stdout, and stderr (circa 1992). Thanks to Miguel Pineiro Jr. + . + +December 8, 2021: + The error handling in closefile and closeall was mangled. Long + standing warnings had been made fatal and some fatal errors went + undetected. Thanks to Miguel Pineiro Jr. . + +Nov 03, 2021: + getline accesses uninitialized data after getrec() + returns 0 on EOF and leaves the contents of buf unchanged. + Thanks to Volodymyr Gubarkov, and Todd C Miller. + +Oct 12, 2021: + The fix for #83 changed the code to insert 2 chars, but the + call to adjbuf just above it only allows for 1 char. This can + cause a heap buffer overflow. July 27, 2021: As per IEEE Std 1003.1-2008, -F "str" is now consistent with -v FS="str" when str is null. Thanks to Warner Losh. July 24, 2021: Fix readrec's definition of a record. This fixes an issue with NetBSD's RS regular expression support that can cause an infinite read loop. Thanks to Miguel Pineiro Jr. Fix regular expression RS ^-anchoring. RS ^-anchoring needs to know if it is reading the first record of a file. This change restores a missing line that was overlooked when porting NetBSD's RS regex functionality. Thanks to Miguel Pineiro Jr. Fix size computation in replace_repeat() for special case REPEAT_WITH_Q. Thanks to Todd C. Miller. February 15, 2021: Small fix so that awk will compile again with g++. Thanks to Arnold Robbins. January 06, 2021: Fix a decision bug with trailing stuff in lib.c:is_valid_number after recent changes. Thanks to Ozan Yigit. December 18, 2020: Fix problems converting inf and NaN values in lib.c:is_valid_number. Enhance number to string conversion to do the right thing for NaN and inf values. Things are now pretty much the same as in gawk. (Found a gawk bug while we're at it.) Added a torture test for these values. Thanks to Arnold Robbins. Allows closing of PR #101. December 15, 2020: Merge PR #99, which gets the right header for strcasecmp. Thanks to GitHub user michaelforney. December 8, 2020: Merge PR #98: Disallow hex data. Allow only +nan, -nan, +inf, -inf (case independent) to give NaN and infinity values. Improve things so that string to double conversion is only done once, yielding something of a speedup. This obviate PR #95. Thanks to Arnold Robbins. December 3, 2020: Fix to argument parsing to avoid printing spurious newlines. Thanks to Todd Miller. Merges PR #97. October 13, 2020: Add casts before all the calls to malloc/calloc/realloc in order to get it to compile with g++. Thanks to Arnold Robbins. August 16, 2020: Additional fixes for DJGPP. Thanks to Eli Zaretskii for the testing. August 7, 2020: Merge PR #93, which adds casts to (void*) for debug prints using the %p format specifier. Thanks to GitHub user YongHaoWu ("Chris") for the fixes. August 4, 2020: In run.c, use non-restartable multibyte routines to attain portability to DJGPP. Should fix Issue 92. Thanks to Albert Wik for the report and to Todd Miller for the suggested fix. July 30, 2020: Merge PRs 88-91 which fix small bugs. Thanks to Todd Miller and Tim van der Molen for the fixes. In order to make life easier, we move exclusively to bison as the parser generator. July 2, 2020: Merge PRs 85 and 86 which fix regressions. Thanks to Tim van der Molen for the fixes. June 25, 2020: Merge PRs 82 and 84. The latter fixes issue #83. Thanks to Todd Miller and awkfan77. June 12, 2020: Clear errno before calling errcheck to avoid any spurious errors left over from previous calls that may have set it. Thanks to Todd Miller for the fix, from PR #80. Fix Issue #78 by allowing \r to follow floating point numbers in lib.c:is_number. Thanks to GitHub user ajcarr for the report and to Arnold Robbins for the fix. June 5, 2020: In fldbld(), make sure that inputFS is set before trying to use it. Thanks to Steffen Nurpmeso for the report. May 5, 2020: Fix checks for compilers that can handle noreturn. Thanks to GitHub user enh-google for pointing it out. Closes Issue #79. April 16, 2020: Handle old compilers that don't support C11 (for noreturn). Thanks to Arnold Robbins. April 5, 2020: Use and noreturn instead of GCC attributes. Thanks to GitHub user awkfan77. Closes PR #77. February 28, 2020: More cleanups from Christos Zoulas: notably backslash continuation inside strings removes the newline and a fix for RS = "^a". Fix for address sanitizer-found problem. Thanks to GitHub user enh-google. February 19, 2020: More small cleanups from Christos Zoulas. February 18, 2020: Additional cleanups from Christos Zoulas. It's no longer necessary to use the -y flag to bison. February 6, 2020: Additional small cleanups from Christos Zoulas. awk is now a little more robust about reporting I/O errors upon exit. January 31, 2020: Merge PR #70, which avoids use of variable length arrays. Thanks to GitHub user michaelforney. Fix issue #60 ({0} in interval expressions doesn't work). Also get all tests working again. Thanks to Arnold Robbins. January 24, 2020: A number of small cleanups from Christos Zoulas. Add the close on exec flag to files/pipes opened for redirection; courtesy of Arnold Robbins. January 19, 2020: If POSIXLY_CORRECT is set in the environment, then sub and gsub use POSIX rules for multiple backslashes. This fixes Issue #66, while maintaining backwards compatibility. January 9, 2020: Input/output errors on closing files are now fatal instead of mere warnings. Thanks to Martijn Dekker . January 5, 2020: Fix a bug in the concatentation of two string constants into one done in the grammar. Fixes GitHub issue #61. Thanks to GitHub user awkfan77 for pointing out the direction for the fix. New test T.concat added to the test suite. Fix a few memory leaks reported by valgrind, as well. December 27, 2019: Fix a bug whereby a{0,3} could match four a's. Thanks to "Anonymous AWK fan" for the report. December 11, 2019: Further printf-related fixes for 32 bit systems. Thanks again to Christos Zoulas. December 8, 2019: Fix the return value of sprintf("%d") on 32 bit systems. Thanks to Jim Lowe for the report and to Christos Zoulas for the fix. November 10, 2019: Convert a number of Boolean integer variables into actual bools. Convert compile_time variable into an enum and simplify some of the related code. Thanks to Arnold Robbins. November 8, 2019: Fix from Ori Bernstein to get UTF-8 characters instead of bytes when FS = "". This is currently the only bit of the One True Awk that understands multibyte characters. From Arnold Robbins, apply some cleanups in the test suite. October 25, 2019: More fixes and cleanups from NetBSD, courtesy of Christos Zoulas. Merges PRs 54 and 55. October 24, 2019: Import second round of code cleanups from NetBSD. Much thanks to Christos Zoulas (GitHub user zoulasc). Merges PR 53. Add an optimization for string concatenation, also from Christos. October 17, 2019: Import code cleanups from NetBSD. Much thanks to Christos Zoulas (GitHub user zoulasc). Merges PR 51. October 6, 2019: Import code from NetBSD awk that implements RS as a regular expression. September 10, 2019: Fixes for various array / memory overruns found via gcc's -fsanitize=unknown. Thanks to Alexander Richardson (GitHub user arichardson). Merges PRs 47 and 48. July 28, 2019: Import grammar optimization from NetBSD: Two string constants concatenated together get turned into a single string. July 26, 2019: Support POSIX-specified C-style escape sequences "\a" (alarm) and "\v" (vertical tab) in command line arguments and regular expressions, further to the support for them in strings added on Apr 9, 1989. These now no longer match as literal "a" and "v" characters (as they don't on other awk implementations). Thanks to Martijn Dekker. July 17, 2019: Pull in a number of code cleanups and minor fixes from Warner Losh's bsd-ota branch. The only user visible change is the use of random(3) as the random number generator. Thanks to Warner Losh for collecting all these fixes in one easy place to get them from. July 16, 2019: Fix field splitting to use FS value as of the time a record was read or assigned to. Thanks to GitHub user Cody Mello (melloc) for the fix. (Merged from his branch, via PR #42.) Updated testdir/T.split per said PR as well. June 24, 2019: Extract awktest.tar into testdir directory. Add some very simple mechanics to the makefile for running the tests and for cleaning up. No changes to awk itself. June 17, 2019: Disallow deleting SYMTAB and its elements, which creates use-after-free bugs. Thanks to GitHub user Cody Mello (melloc) for the fix. (Merged from PR #43.) June 5, 2019: Allow unmatched right parenthesis in a regular expression to be treated literally. Fixes Issue #40. Thanks to GitHub user Warner Losh (bsdimp) for the report. Thanks to Arnold Robbins for the fix. May 29,2019: Fix check for command line arguments to no longer require that first character after '=' not be another '='. Reverts change of August 11, 1989. Thanks to GitHub user Jamie Landeg Jones for pointing out the issue; from Issue #38. Apr 7, 2019: Update awktest.tar(p.50) to use modern options to sort. Needed for Android development. Thanks to GitHub user mohd-akram (Mohamed Akram). From Issue #33. Mar 12, 2019: Added very simplistic support for cross-compiling in the makefile. We are NOT going to go in the direction of the autotools, though. Thanks to GitHub user nee-san for the basic change. (Merged from PR #34.) Mar 5, 2019: Added support for POSIX-standard interval expressions (a.k.a. bounds, a.k.a. repetition expressions) in regular expressions, backported (via NetBSD) from Apple awk-24 (20070501). Thanks to Martijn Dekker for the port. (Merged from PR #30.) Mar 3, 2019: Merge PRs as follows: #12: Avoid undefined behaviour when using ctype(3) functions in relex(). Thanks to GitHub user iamleot. #31: Make getline handle numeric strings, and update FIXES. Thanks - to GitHub user arnoldrobbins. + to GitHub user Arnold Robbins (arnoldrobbins) #32: maketab: support build systems with read-only source. Thanks to GitHub user enh. Jan 25, 2019: Make getline handle numeric strings properly in all cases. (Thanks, Arnold.) Jan 21, 2019: Merged a number of small fixes from GitHub pull requests. Thanks to GitHub users Arnold Robbins (arnoldrobbins), Cody Mello (melloc) and Christoph Junghans (junghans). PR numbers: 13-21, 23, 24, 27. Oct 25, 2018: Added test in maketab.c to prevent generating a proctab entry for YYSTYPE_IS_DEFINED. It was harmless but some gcc settings generated a warning message. Thanks to Nan Xiao for report. Aug 27, 2018: Disallow '$' in printf formats; arguments evaluated in order and printed in order. Added some casts to silence warnings on debugging printfs. (Thanks, Arnold.) Aug 23, 2018: A long list of fixes courtesy of Arnold Robbins, to whom profound thanks. 1. ofs-rebuild: OFS value used to rebuild the record was incorrect. Fixed August 19, 2014. Revised fix August 2018. 2. system-status: Instead of a floating-point division by 256, use the wait(2) macros to create a reasonable exit status. Fixed March 12, 2016. 3. space: Use provided xisblank() function instead of ispace() for matching [[:blank:]]. 4. a-format: Add POSIX standard %a and %A to supported formats. Check at runtime that this format is available. 5. decr-NF: Decrementing NF did not change $0. This is a decades-old bug. There are interactions with the old and new value of OFS as well. Most of the fix came from the NetBSD awk. 6. string-conv: String conversions of scalars were sticky. Once a conversion to string happened, even with OFMT, that value was used until a new numeric value was assigned, even if OFMT differed from CONVFMT, and also if CONVFMT changed. 7. unary-plus: Unary plus on a string constant returned the string. Instead, it should convert the value to numeric and give that value. Also added Arnold's tests for these to awktest.tar as T.arnold. Aug 15, 2018: fixed mangled awktest.tar (thanks, Arnold), posted all current (very minor) fixes to github / onetrueawk Jun 7, 2018: (yes, a long layoff) Updated some broken tests (beebe.tar, T.lilly) [thanks to Arnold Robbins] Mar 26, 2015: buffer overflow in error reporting; thanks to tobias ulmer and john-mark gurney for spotting it and the fix. Feb 4, 2013: cleaned up a handful of tests that didn't seem to actually test for correct behavior: T.latin1, T.gawk. Jan 5, 2013: added ,NULL initializer to static Cells in run.c; not really needed but cleaner. Thanks to Michael Bombardieri. Dec 20, 2012: fiddled makefile to get correct yacc and bison flags. pick yacc (linux) or bison (mac) as necessary. added __attribute__((__noreturn__)) to a couple of lines in proto.h, to silence someone's enthusiastic checker. fixed obscure call by value bug in split(a[1],a) reported on 9fans. the management of temporary values is just a mess; i took a shortcut by making an extra string copy. thanks to paul patience and arnold robbins for passing it on and for proposed patches. tiny fiddle in setfval to eliminate -0 results in T.expr, which has irritated me for 20+ years. Aug 10, 2011: another fix to avoid core dump with delete(ARGV); again, many thanks to ruslan ermilov. Aug 7, 2011: split(s, a, //) now behaves the same as split(s, a, "") Jun 12, 2011: /pat/, \n /pat/ {...} is now legal, though bad style to use. added checks to new -v code that permits -vnospace; thanks to ruslan ermilov for spotting this and providing the patch. removed fixed limit on number of open files; thanks to aleksey cheusov and christos zoulos. fixed day 1 bug that resurrected deleted elements of ARGV when used as filenames (in lib.c). minor type fiddles to make gcc -Wall -pedantic happier (but not totally so); turned on -fno-strict-aliasing in makefile. May 6, 2011: added #ifdef for isblank. now allows -ffoo as well as -f foo arguments. (thanks, ruslan) May 1, 2011: after advice from todd miller, kevin lo, ruslan ermilov, and arnold robbins, changed srand() to return the previous seed (which is 1 on the first call of srand). the seed is an Awkfloat internally though converted to unsigned int to pass to the library srand(). thanks, everyone. fixed a subtle (and i hope low-probability) overflow error in fldbld, by adding space for one extra \0. thanks to robert bassett for spotting this one and providing a fix. removed the files related to compilation on windows. i no longer have anything like a current windows environment, so i can't test any of it. May 23, 2010: fixed long-standing overflow bug in run.c; many thanks to nelson beebe for spotting it and providing the fix. fixed bug that didn't parse -vd=1 properly; thanks to santiago vila for spotting it. Feb 8, 2010: i give up. replaced isblank with isspace in b.c; there are no consistent header files. Nov 26, 2009: fixed a long-standing issue with when FS takes effect. a change to FS is now noticed immediately for subsequent splits. changed the name getline() to awkgetline() to avoid yet another name conflict somewhere. Feb 11, 2009: temporarily for now defined HAS_ISBLANK, since that seems to be the best way through the thicket. isblank arrived in C99, but seems to be arriving at different systems at different times. Oct 8, 2008: fixed typo in b.c that set tmpvec wrongly. no one had ever run into the problem, apparently. thanks to alistair crooks. Oct 23, 2007: minor fix in lib.c: increase inputFS to 100, change malloc for fields to n+1. fixed memory fault caused by out of order test in setsval. thanks to david o'brien, freebsd, for both fixes. May 1, 2007: fiddle in makefile to fix for BSD make; thanks to igor sobrado. Mar 31, 2007: fixed some null pointer refs calling adjbuf. Feb 21, 2007: fixed a bug in matching the null RE in sub and gsub. thanks to al aho who actually did the fix (in b.c), and to wolfgang seeberg for finding it and providing a very compact test case. fixed quotation in b.c; thanks to Hal Pratt and the Princeton Dante Project. removed some no-effect asserts in run.c. fiddled maketab.c to not complain about bison-generated values. removed the obsolete -V argument; fixed --version to print the version and exit. fixed wording and an outright error in the usage message; thanks to igor sobrado and jason mcintyre. fixed a bug in -d that caused core dump if no program followed. Jan 1, 2007: dropped mac.code from makefile; there are few non-MacOSX mac's these days. Jan 17, 2006: system() not flagged as unsafe in the unadvertised -safe option. found it while enhancing tests before shipping the ;login: article. practice what you preach. removed the 9-years-obsolete -mr and -mf flags. added -version and --version options. core dump on linux with BEGIN {nextfile}, now fixed. removed some #ifdef's in run.c and lex.c that appear to no longer be necessary. Apr 24, 2005: modified lib.c so that values of $0 et al are preserved in the END block, apparently as required by posix. thanks to havard eidnes for the report and code. Jan 14, 2005: fixed infinite loop in parsing, originally found by brian tsang. thanks to arnold robbins for a suggestion that started me rethinking it. Dec 31, 2004: prevent overflow of -f array in main, head off potential error in call of SYNTAX(), test malloc return in lib.c, all with thanks to todd miller. Dec 22, 2004: cranked up size of NCHARS; coverity thinks it can be overrun with smaller size, and i think that's right. added some assertions to b.c to catch places where it might overrun. the RE code is still fragile. Dec 5, 2004: fixed a couple of overflow problems with ridiculous field numbers: e.g., print $(2^32-1). thanks to ruslan ermilov, giorgos keramidas and david o'brien at freebsd.org for patches. this really should be re-done from scratch. Nov 21, 2004: fixed another 25-year-old RE bug, in split. it's another failure to (re-)initialize. thanks to steve fisher for spotting this and providing a good test case. Nov 22, 2003: fixed a bug in regular expressions that dates (so help me) from 1977; it's been there from the beginning. an anchored longest match that was longer than the number of states triggered a failure to initialize the machine properly. many thanks to moinak ghosh for not only finding this one but for providing a fix, in some of the most mysterious code known to man. fixed a storage leak in call() that appears to have been there since 1983 or so -- a function without an explicit return that assigns a string to a parameter leaked a Cell. thanks to moinak ghosh for spotting this very subtle one. Jul 31, 2003: fixed, thanks to andrey chernov and ruslan ermilov, a bug in lex.c that mis-handled the character 255 in input. (it was being compared to EOF with a signed comparison.) Jul 29, 2003: fixed (i think) the long-standing botch that included the beginning of line state ^ for RE's in the set of valid characters; this led to a variety of odd problems, including failure to properly match certain regular expressions in non-US locales. thanks to ruslan for keeping at this one. Jul 28, 2003: n-th try at getting internationalization right, with thanks to volker kiefel, arnold robbins and ruslan ermilov for advice, though they should not be blamed for the outcome. according to posix, "." is the radix character in programs and command line arguments regardless of the locale; otherwise, the locale should prevail for input and output of numbers. so it's intended to work that way. i have rescinded the attempt to use strcoll in expanding shorthands in regular expressions (cclenter). its properties are much too surprising; for example [a-c] matches aAbBc in locale en_US but abBcC in locale fr_CA. i can see how this might arise by implementation but i cannot explain it to a human user. (this behavior can be seen in gawk as well; we're leaning on the same library.) the issue appears to be that strcoll is meant for sorting, where merging upper and lower case may make sense (though note that unix sort does not do this by default either). it is not appropriate for regular expressions, where the goal is to match specific patterns of characters. in any case, the notations [:lower:], etc., are available in awk, and they are more likely to work correctly in most locales. a moratorium is hereby declared on internationalization changes. i apologize to friends and colleagues in other parts of the world. i would truly like to get this "right", but i don't know what that is, and i do not want to keep making changes until it's clear. Jul 4, 2003: fixed bug that permitted non-terminated RE, as in "awk /x". Jun 1, 2003: subtle change to split: if source is empty, number of elems is always 0 and the array is not set. Mar 21, 2003: added some parens to isblank, in another attempt to make things internationally portable. Mar 14, 2003: the internationalization changes, somewhat modified, are now reinstated. in theory awk will now do character comparisons and case conversions in national language, but "." will always be the decimal point separator on input and output regardless of national language. isblank(){} has an #ifndef. this no longer compiles on windows: LC_MESSAGES isn't defined in vc6++. fixed subtle behavior in field and record splitting: if FS is a single character and RS is not empty, \n is NOT a separator. this tortuous reading is found in the awk book; behavior now matches gawk and mawk. Dec 13, 2002: for the moment, the internationalization changes of nov 29 are rolled back -- programs like x = 1.2 don't work in some locales, because the parser is expecting x = 1,2. until i understand this better, this will have to wait. Nov 29, 2002: modified b.c (with tiny changes in main and run) to support locales, using strcoll and iswhatever tests for posix character classes. thanks to ruslan ermilov (ru@freebsd.org) for code. the function isblank doesn't seem to have propagated to any header file near me, so it's there explicitly. not properly tested on non-ascii character sets by me. Jun 28, 2002: modified run/format() and tran/getsval() to do a slightly better job on using OFMT for output from print and CONVFMT for other number->string conversions, as promised by posix and done by gawk and mawk. there are still places where it doesn't work right if CONVFMT is changed; by then the STR attribute of the variable has been irrevocably set. thanks to arnold robbins for code and examples. fixed subtle bug in format that could get core dump. thanks to Jaromir Dolecek for finding and fixing. minor cleanup in run.c / format() at the same time. added some tests for null pointers to debugging printf's, which were never intended for external consumption. thanks to dave kerns (dkerns@lucent.com) for pointing this out. GNU compatibility: an empty regexp matches anything (thanks to dag-erling smorgrav, des@ofug.org). subject to reversion if this does more harm than good. pervasive small changes to make things more const-correct, as reported by gcc's -Wwrite-strings. as it says in the gcc manual, this may be more nuisance than useful. provoked by a suggestion and code from arnaud desitter, arnaud@nimbus.geog.ox.ac.uk minor documentation changes to note that this now compiles out of the box on Mac OS X. Feb 10, 2002: changed types in posix chars structure to quiet solaris cc. Jan 1, 2002: fflush() or fflush("") flushes all files and pipes. length(arrayname) returns number of elements; thanks to arnold robbins for suggestion. added a makefile.win to make it easier to build on windows. based on dan allen's buildwin.bat. Nov 16, 2001: added support for posix character class names like [:digit:], which are not exactly shorter than [0-9] and perhaps no more portable. thanks to dag-erling smorgrav for code. Feb 16, 2001: removed -m option; no longer needed, and it was actually broken (noted thanks to volker kiefel). Feb 10, 2001: fixed an appalling bug in gettok: any sequence of digits, +,-, E, e, and period was accepted as a valid number if it started with a period. this would never have happened with the lex version. other 1-character botches, now fixed, include a bare $ and a bare " at the end of the input. Feb 7, 2001: more (const char *) casts in b.c and tran.c to silence warnings. Nov 15, 2000: fixed a bug introduced in august 1997 that caused expressions like $f[1] to be syntax errors. thanks to arnold robbins for noticing this and providing a fix. Oct 30, 2000: fixed some nextfile bugs: not handling all cases. thanks to arnold robbins for pointing this out. new regressions added. close() is now a function. it returns whatever the library fclose returns, and -1 for closing a file or pipe that wasn't opened. Sep 24, 2000: permit \n explicitly in character classes; won't work right if comes in as "[\n]" but ok as /[\n]/, because of multiple processing of \'s. thanks to arnold robbins. July 5, 2000: minor fiddles in tran.c to keep compilers happy about uschar. thanks to norman wilson. May 25, 2000: yet another attempt at making 8-bit input work, with another band-aid in b.c (member()), and some (uschar) casts to head off potential errors in subscripts (like isdigit). also changed HAT to NCHARS-2. thanks again to santiago vila. changed maketab.c to ignore apparently out of range definitions instead of halting; new freeBSD generates one. thanks to jon snader for pointing out the problem. May 2, 2000: fixed an 8-bit problem in b.c by making several char*'s into unsigned char*'s. not clear i have them all yet. thanks to Santiago Vila for the bug report. Apr 21, 2000: finally found and fixed a memory leak in function call; it's been there since functions were added ~1983. thanks to jon bentley for the test case that found it. added test in envinit to catch environment "variables" with names beginning with '='; thanks to Berend Hasselman. Jul 28, 1999: added test in defn() to catch function foo(foo), which otherwise recurses until core dump. thanks to arnold robbins for noticing this. Jun 20, 1999: added *bp in gettok in lex.c; appears possible to exit function without terminating the string. thanks to russ cox. Jun 2, 1999: added function stdinit() to run to initialize files[] array, in case stdin, etc., are not constants; some compilers care. May 10, 1999: replaced the ERROR ... FATAL, etc., macros with functions based on vprintf, to avoid problems caused by overrunning fixed-size errbuf array. thanks to ralph corderoy for the impetus, and for pointing out a string termination bug in qstring as well. Apr 21, 1999: fixed bug that caused occasional core dumps with commandline variable with value ending in \. (thanks to nelson beebe for the test case.) Apr 16, 1999: with code kindly provided by Bruce Lilly, awk now parses /=/ and similar constructs more sensibly in more places. Bruce also provided some helpful test cases. Apr 5, 1999: changed true/false to True/False in run.c to make it easier to compile with C++. Added some casts on malloc and realloc to be honest about casts; ditto. changed ltype int to long in struct rrow to reduce some 64-bit complaints; other changes scattered throughout for the same purpose. thanks to Nelson Beebe for these portability improvements. removed some horrible pointer-int casting in b.c and elsewhere by adding ptoi and itonp to localize the casts, which are all benign. fixed one incipient bug that showed up on sgi in 64-bit mode. reset lineno for new source file; include filename in error message. also fixed line number error in continuation lines. (thanks to Nelson Beebe for both of these.) Mar 24, 1999: Nelson Beebe notes that irix 5.3 yacc dies with a bogus error; use a newer version or switch to bison, since sgi is unlikely to fix it. Mar 5, 1999: changed isnumber to is_number to avoid the problem caused by versions of ctype.h that include the name isnumber. distribution now includes a script for building on a Mac, thanks to Dan Allen. Feb 20, 1999: fixed memory leaks in run.c (call) and tran.c (setfval). thanks to Stephen Nutt for finding these and providing the fixes. Jan 13, 1999: replaced srand argument by (unsigned int) in run.c; avoids problem on Mac and potentially on Unix & Windows. thanks to Dan Allen. added a few (int) casts to silence useless compiler warnings. e.g., errorflag= in run.c jump(). added proctab.c to the bundle outout; one less thing to have to compile out of the box. added calls to _popen and _pclose to the win95 stub for pipes (thanks to Steve Adams for this helpful suggestion). seems to work, though properties are not well understood by me, and it appears that under some circumstances the pipe output is truncated. Be careful. Oct 19, 1998: fixed a couple of bugs in getrec: could fail to update $0 after a getline var; because inputFS wasn't initialized, could split $0 on every character, a misleading diversion. fixed caching bug in makedfa: LRU was actually removing least often used. thanks to ross ridge for finding these, and for providing great bug reports. May 12, 1998: fixed potential bug in readrec: might fail to update record pointer after growing. thanks to dan levy for spotting this and suggesting the fix. Mar 12, 1998: added -V to print version number and die. [notify dave kerns, dkerns@dacsoup.ih.lucent.com] Feb 11, 1998: subtle silent bug in lex.c: if the program ended with a number longer than 1 digit, part of the input would be pushed back and parsed again because token buffer wasn't terminated right. example: awk 'length($0) > 10'. blush. at least i found it myself. Aug 31, 1997: s/adelete/awkdelete/: SGI uses this in malloc.h. thanks to nelson beebe for pointing this one out. Aug 21, 1997: fixed some bugs in sub and gsub when replacement includes \\. this is a dark, horrible corner, but at least now i believe that the behavior is the same as gawk and the intended posix standard. thanks to arnold robbins for advice here. Aug 9, 1997: somewhat regretfully, replaced the ancient lex-based lexical analyzer with one written in C. it's longer, generates less code, and more portable; the old one depended too much on mysterious properties of lex that were not preserved in other environments. in theory these recognize the same language. now using strtod to test whether a string is a number, instead of the convoluted original function. should be more portable and reliable if strtod is implemented right. removed now-pointless optimization in makefile that tries to avoid recompilation when awkgram.y is changed but symbols are not. removed most fixed-size arrays, though a handful remain, some of which are unchecked. you have been warned. Aug 4, 1997: with some trepidation, replaced the ancient code that managed fields and $0 in fixed-size arrays with arrays that grow on demand. there is still some tension between trying to make this run fast and making it clean; not sure it's right yet. the ill-conceived -mr and -mf arguments are now useful only for debugging. previous dynamic string code removed. numerous other minor cleanups along the way. Jul 30, 1997: using code provided by dan levy (to whom profuse thanks), replaced fixed-size arrays and awkward kludges by a fairly uniform mechanism to grow arrays as needed for printf, sub, gsub, etc. Jul 23, 1997: falling off the end of a function returns "" and 0, not 0. thanks to arnold robbins. Jun 17, 1997: replaced several fixed-size arrays by dynamically-created ones in run.c; added overflow tests to some previously unchecked cases. getline, toupper, tolower. getline code is still broken in that recursive calls may wind up using the same space. [fixed later] increased RECSIZE to 8192 to push problems further over the horizon. added \r to \n as input line separator for programs, not data. damn CRLFs. modified format() to permit explicit printf("%c", 0) to include a null byte in output. thanks to ken stailey for the fix. added a "-safe" argument that disables file output (print >, print >>), process creation (cmd|getline, print |, system), and access to the environment (ENVIRON). this is a first approximation to a "safe" version of awk, but don't rely on it too much. thanks to joan feigenbaum and matt blaze for the inspiration long ago. Jul 8, 1996: fixed long-standing bug in sub, gsub(/a/, "\\\\&"); thanks to ralph corderoy. Jun 29, 1996: fixed awful bug in new field splitting; didn't get all the places where input was done. Jun 28, 1996: changed field-splitting to conform to posix definition: fields are split using the value of FS at the time of input; it used to be the value when the field or NF was first referred to, a much less predictable definition. thanks to arnold robbins for encouragement to do the right thing. May 28, 1996: fixed appalling but apparently unimportant bug in parsing octal numbers in reg exprs. explicit hex in reg exprs now limited to 2 chars: \xa, \xaa. May 27, 1996: cleaned up some declarations so gcc -Wall is now almost silent. makefile now includes backup copies of ytab.c and lexyy.c in case one makes before looking; it also avoids recreating lexyy.c unless really needed. s/aprintf/awkprint, s/asprintf/awksprintf/ to avoid some name clashes with unwisely-written header files. thanks to jeffrey friedl for several of these. May 26, 1996: an attempt to rationalize the (unsigned) char issue. almost all instances of unsigned char have been removed; the handful of places in b.c where chars are used as table indices have been hand-crafted. added some latin-1 tests to the regression, but i'm not confident; none of my compilers seem to care much. thanks to nelson beebe for pointing out some others that do care. May 2, 1996: removed all register declarations. enhanced split(), as in gawk, etc: split(s, a, "") splits s into a[1]...a[length(s)] with each character a single element. made the same changes for field-splitting if FS is "". added nextfile, as in gawk: causes immediate advance to next input file. (thanks to arnold robbins for inspiration and code). small fixes to regexpr code: can now handle []], [[], and variants; [] is now a syntax error, rather than matching everything; [z-a] is now empty, not z. far from complete or correct, however. (thanks to jeffrey friedl for pointing out some awful behaviors.) Apr 29, 1996: replaced uchar by uschar everywhere; apparently some compilers usurp this name and this causes conflicts. fixed call to time in run.c (bltin); arg is time_t *. replaced horrible pointer/long punning in b.c by a legitimate union. should be safer on 64-bit machines and cleaner everywhere. (thanks to nelson beebe for pointing out some of these problems.) replaced nested comments by #if 0...#endif in run.c, lib.c. removed getsval, setsval, execute macros from run.c and lib.c. machines are 100x faster than they were when these macros were first used. revised filenames: awk.g.y => awkgram.y, awk.lx.l => awklex.l, y.tab.[ch] => ytab.[ch], lex.yy.c => lexyy.c, all in the aid of portability to nameless systems. "make bundle" now includes yacc and lex output files for recipients who don't have yacc or lex. Aug 15, 1995: initialized Cells in setsymtab more carefully; some fields were not set. (thanks to purify, all of whose complaints i think i now understand.) fixed at least one error in gsub that looked at -1-th element of an array when substituting for a null match (e.g., $). delete arrayname is now legal; it clears the elements but leaves the array, which may not be the right behavior. modified makefile: my current make can't cope with the test used to avoid unnecessary yacc invocations. Jul 17, 1995: added dynamically growing strings to awk.lx.l and b.c to permit regular expressions to be much bigger. the state arrays can still overflow. Aug 24, 1994: detect duplicate arguments in function definitions (mdm). May 11, 1994: trivial fix to printf to limit string size in sub(). Apr 22, 1994: fixed yet another subtle self-assignment problem: $1 = $2; $1 = $1 clobbered $1. Regression tests now use private echo, to avoid quoting problems. Feb 2, 1994: changed error() to print line number as %d, not %g. Jul 23, 1993: cosmetic changes: increased sizes of some arrays, reworded some error messages. added CONVFMT as in posix (just replaced OFMT in getsval) FILENAME is now "" until the first thing that causes a file to be opened. Nov 28, 1992: deleted yyunput and yyoutput from proto.h; different versions of lex give these different declarations. May 31, 1992: added -mr N and -mf N options: more record and fields. these really ought to adjust automatically. cleaned up some error messages; "out of space" now means malloc returned NULL in all cases. changed rehash so that if it runs out, it just returns; things will continue to run slow, but maybe a bit longer. Apr 24, 1992: remove redundant close of stdin when using -f -. got rid of core dump with -d; awk -d just prints date. Apr 12, 1992: added explicit check for /dev/std(in,out,err) in redirection. unlike gawk, no /dev/fd/n yet. added (file/pipe) builtin. hard to test satisfactorily. not posix. Feb 20, 1992: recompile after abortive changes; should be unchanged. Dec 2, 1991: die-casting time: converted to ansi C, installed that. Nov 30, 1991: fixed storage leak in freefa, failing to recover [N]CCL. thanks to Bill Jones (jones@cs.usask.ca) Nov 19, 1991: use RAND_MAX instead of literal in builtin(). Nov 12, 1991: cranked up some fixed-size arrays in b.c, and added a test for overflow in penter. thanks to mark larsen. Sep 24, 1991: increased buffer in gsub. a very crude fix to a general problem. and again on Sep 26. Aug 18, 1991: enforce variable name syntax for commandline variables: has to start with letter or _. Jul 27, 1991: allow newline after ; in for statements. Jul 21, 1991: fixed so that in self-assignment like $1=$1, side effects like recomputing $0 take place. (this is getting subtle.) Jun 30, 1991: better test for detecting too-long output record. Jun 2, 1991: better defense against very long printf strings. made break and continue illegal outside of loops. May 13, 1991: removed extra arg on gettemp, tempfree. minor error message rewording. May 6, 1991: fixed silly bug in hex parsing in hexstr(). removed an apparently unnecessary test in isnumber(). warn about weird printf conversions. fixed unchecked array overwrite in relex(). changed for (i in array) to access elements in sorted order. then unchanged it -- it really does run slower in too many cases. left the code in place, commented out. Feb 10, 1991: check error status on all writes, to avoid banging on full disks. Jan 28, 1991: awk -f - reads the program from stdin. Jan 11, 1991: failed to set numeric state on $0 in cmd|getline context in run.c. Nov 2, 1990: fixed sleazy test for integrality in getsval; use modf. Oct 29, 1990: fixed sleazy buggy code in lib.c that looked (incorrectly) for too long input lines. Oct 14, 1990: fixed the bug on p. 198 in which it couldn't deduce that an argument was an array in some contexts. replaced the error message in intest() by code that damn well makes it an array. Oct 8, 1990: fixed horrible bug: types and values were not preserved in some kinds of self-assignment. (in assign().) Aug 24, 1990: changed NCHARS to 256 to handle 8-bit characters in strings presented to match(), etc. Jun 26, 1990: changed struct rrow (awk.h) to use long instead of int for lval, since cfoll() stores a pointer in it. now works better when int's are smaller than pointers! May 6, 1990: AVA fixed the grammar so that ! is uniformly of the same precedence as unary + and -. This renders illegal some constructs like !x=y, which now has to be parenthesized as !(x=y), and makes others work properly: !x+y is (!x)+y, and x!y is x !y, not two pattern-action statements. (These problems were pointed out by Bob Lenk of Posix.) Added \x to regular expressions (already in strings). Limited octal to octal digits; \8 and \9 are not octal. Centralized the code for parsing escapes in regular expressions. Added a bunch of tests to T.re and T.sub to verify some of this. Feb 9, 1990: fixed null pointer dereference bug in main.c: -F[nothing]. sigh. restored srand behavior: it returns the current seed. Jan 18, 1990: srand now returns previous seed value (0 to start). Jan 5, 1990: fix potential problem in tran.c -- something was freed, then used in freesymtab. Oct 18, 1989: another try to get the max number of open files set with relatively machine-independent code. small fix to input() in case of multiple reads after EOF. Oct 11, 1989: FILENAME is now defined in the BEGIN block -- too many old programs broke. "-" means stdin in getline as well as on the commandline. added a bunch of casts to the code to tell the truth about char * vs. unsigned char *, a right royal pain. added a setlocale call to the front of main, though probably no one has it usefully implemented yet. Aug 24, 1989: removed redundant relational tests against nullnode if parse tree already had a relational at that point. Aug 11, 1989: fixed bug: commandline variable assignment has to look like var=something. (consider the man page for =, in file =.1) changed number of arguments to functions to static arrays to avoid repeated malloc calls. Aug 2, 1989: restored -F (space) separator Jul 30, 1989: added -v x=1 y=2 ... for immediate commandline variable assignment; done before the BEGIN block for sure. they have to precede the program if the program is on the commandline. Modified Aug 2 to require a separate -v for each assignment. Jul 10, 1989: fixed ref-thru-zero bug in environment code in tran.c Jun 23, 1989: add newline to usage message. Jun 14, 1989: added some missing ansi printf conversion letters: %i %X %E %G. no sensible meaning for h or L, so they may not do what one expects. made %* conversions work. changed x^y so that if n is a positive integer, it's done by explicit multiplication, thus achieving maximum accuracy. (this should be done by pow() but it seems not to be locally.) done to x ^= y as well. Jun 4, 1989: ENVIRON array contains environment: if shell variable V=thing, ENVIRON["V"] is "thing" multiple -f arguments permitted. error reporting is naive. (they were permitted before, but only the last was used.) fixed a really stupid botch in the debugging macro dprintf fixed order of evaluation of commandline assignments to match what the book claims: an argument of the form x=e is evaluated at the time it would have been opened if it were a filename (p 63). this invalidates the suggested answer to ex 4-1 (p 195). removed some code that permitted -F (space) fieldseparator, since it didn't quite work right anyway. (restored aug 2) Apr 27, 1989: Line number now accumulated correctly for comment lines. Apr 26, 1989: Debugging output now includes a version date, if one compiles it into the source each time. Apr 9, 1989: Changed grammar to prohibit constants as 3rd arg of sub and gsub; prevents class of overwriting-a-constant errors. (Last one?) This invalidates the "banana" example on page 43 of the book. Added \a ("alert"), \v (vertical tab), \xhhh (hexadecimal), as in ANSI, for strings. Rescinded the sloppiness that permitted non-octal digits in \ooo. Warning: not all compilers and libraries will be able to deal with \x correctly. Jan 9, 1989: Fixed bug that caused tempcell list to contain a duplicate. The fix is kludgy. Dec 17, 1988: Catches some more commandline errors in main. Removed redundant decl of modf in run.c (confuses some compilers). Warning: there's no single declaration of malloc, etc., in awk.h that seems to satisfy all compilers. Dec 7, 1988: Added a bit of code to error printing to avoid printing nulls. (Not clear that it actually would.) Nov 27, 1988: With fear and trembling, modified the grammar to permit multiple pattern-action statements on one line without an explicit separator. By definition, this capitulation to the ghost of ancient implementations remains undefined and thus subject to change without notice or apology. DO NOT COUNT ON IT. Oct 30, 1988: Fixed bug in call() that failed to recover storage. A warning is now generated if there are more arguments in the call than in the definition (in lieu of fixing another storage leak). Oct 20, 1988: Fixed %c: if expr is numeric, use numeric value; otherwise print 1st char of string value. still doesn't work if the value is 0 -- won't print \0. Added a few more checks for running out of malloc. Oct 12, 1988: Fixed bug in call() that freed local arrays twice. Fixed to handle deletion of non-existent array right; complains about attempt to delete non-array element. Sep 30, 1988: Now guarantees to evaluate all arguments of built-in functions, as in C; the appearance is that arguments are evaluated before the function is called. Places affected are sub (gsub was ok), substr, printf, and all the built-in arithmetic functions in bltin(). A warning is generated if a bltin() is called with the wrong number of arguments. This requires changing makeprof on p167 of the book. Aug 23, 1988: setting FILENAME in BEGIN caused core dump, apparently because it was freeing space not allocated by malloc. July 24, 1988: fixed egregious error in toupper/tolower functions. still subject to rescinding, however. July 2, 1988: flush stdout before opening file or pipe July 2, 1988: performance bug in b.c/cgoto(): not freeing some sets of states. partial fix only right now, and the number of states increased to make it less obvious. June 1, 1988: check error status on close May 28, 1988: srand returns seed value it's using. see 1/18/90 May 22, 1988: Removed limit on depth of function calls. May 10, 1988: Fixed lib.c to permit _ in commandline variable names. Mar 25, 1988: main.c fixed to recognize -- as terminator of command- line options. Illegal options flagged. Error reporting slightly cleaned up. Dec 2, 1987: Newer C compilers apply a strict scope rule to extern declarations within functions. Two extern declarations in lib.c and tran.c have been moved to obviate this problem. Oct xx, 1987: Reluctantly added toupper and tolower functions. Subject to rescinding without notice. Sep 17, 1987: Error-message printer had printf(s) instead of printf("%s",s); got core dumps when the message included a %. Sep 12, 1987: Very long printf strings caused core dump; fixed aprintf, asprintf, format to catch them. Can still get a core dump in printf itself. diff --git a/contrib/one-true-awk/README.md b/contrib/one-true-awk/README.md index 76ae3d48c983..a41fb3c3b128 100644 --- a/contrib/one-true-awk/README.md +++ b/contrib/one-true-awk/README.md @@ -1,123 +1,149 @@ # The One True Awk This is the version of `awk` described in _The AWK Programming Language_, -by Al Aho, Brian Kernighan, and Peter Weinberger -(Addison-Wesley, 1988, ISBN 0-201-07981-X). +Second Edition, by Al Aho, Brian Kernighan, and Peter Weinberger +(Addison-Wesley, 2024, ISBN-13 978-0138269722, ISBN-10 0138269726). + +## What's New? ## + +This version of Awk handles UTF-8 and comma-separated values (CSV) input. + +### Strings ### + +Functions that process strings now count Unicode code points, not bytes; +this affects `length`, `substr`, `index`, `match`, `split`, +`sub`, `gsub`, and others. Note that code +points are not necessarily characters. + +UTF-8 sequences may appear in literal strings and regular expressions. +Aribtrary characters may be included with `\u` followed by 1 to 8 hexadecimal digits. + +### Regular expressions ### + +Regular expressions may include UTF-8 code points, including `\u`. + +### CSV ### + +The option `--csv` turns on CSV processing of input: +fields are separated by commas, fields may be quoted with +double-quote (`"`) characters, quoted fields may contain embedded newlines. +Double-quotes in fields have to be doubled and enclosed in quoted fields. +In CSV mode, `FS` is ignored. + +If no explicit separator argument is provided, +field-splitting in `split` is determined by CSV mode. ## Copyright Copyright (C) Lucent Technologies 1997
All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ## Distribution and Reporting Problems Changes, mostly bug fixes and occasional enhancements, are listed in `FIXES`. If you distribute this code further, please please please distribute `FIXES` with it. If you find errors, please report them -to bwk@cs.princeton.edu. +to the current maintainer, ozan.yigit@gmail.com. Please _also_ open an issue in the GitHub issue tracker, to make it easy to track issues. Thanks. ## Submitting Pull Requests Pull requests are welcome. Some guidelines: * Please do not use functions or facilities that are not standard (e.g., `strlcpy()`, `fpurge()`). * Please run the test suite and make sure that your changes pass before posting the pull request. To do so: 1. Save the previous version of `awk` somewhere in your path. Call it `nawk` (for example). 1. Run `oldawk=nawk make check > check.out 2>&1`. 1. Search for `BAD` or `error` in the result. In general, look over it manually to make sure there are no errors. * Please create the pull request with a request to merge into the `staging` branch instead of into the `master` branch. This allows us to do testing, and to make any additional edits or changes after the merge but before merging to `master`. ## Building The program itself is created by make which should produce a sequence of messages roughly like this: - yacc -d awkgram.y - conflicts: 43 shift/reduce, 85 reduce/reduce - mv y.tab.c ytab.c - mv y.tab.h ytab.h - cc -c ytab.c - cc -c b.c - cc -c main.c - cc -c parse.c - cc maketab.c -o maketab - ./maketab >proctab.c - cc -c proctab.c - cc -c tran.c - cc -c lib.c - cc -c run.c - cc -c lex.c - cc ytab.o b.o main.o parse.o proctab.o tran.o lib.o run.o lex.o -lm + bison -d awkgram.y + awkgram.y: warning: 44 shift/reduce conflicts [-Wconflicts-sr] + awkgram.y: warning: 85 reduce/reduce conflicts [-Wconflicts-rr] + awkgram.y: note: rerun with option '-Wcounterexamples' to generate conflict counterexamples + gcc -g -Wall -pedantic -Wcast-qual -O2 -c -o awkgram.tab.o awkgram.tab.c + gcc -g -Wall -pedantic -Wcast-qual -O2 -c -o b.o b.c + gcc -g -Wall -pedantic -Wcast-qual -O2 -c -o main.o main.c + gcc -g -Wall -pedantic -Wcast-qual -O2 -c -o parse.o parse.c + gcc -g -Wall -pedantic -Wcast-qual -O2 maketab.c -o maketab + ./maketab awkgram.tab.h >proctab.c + gcc -g -Wall -pedantic -Wcast-qual -O2 -c -o proctab.o proctab.c + gcc -g -Wall -pedantic -Wcast-qual -O2 -c -o tran.o tran.c + gcc -g -Wall -pedantic -Wcast-qual -O2 -c -o lib.o lib.c + gcc -g -Wall -pedantic -Wcast-qual -O2 -c -o run.o run.c + gcc -g -Wall -pedantic -Wcast-qual -O2 -c -o lex.o lex.c + gcc -g -Wall -pedantic -Wcast-qual -O2 awkgram.tab.o b.o main.o parse.o proctab.o tran.o lib.o run.o lex.o -lm This produces an executable `a.out`; you will eventually want to move this to some place like `/usr/bin/awk`. If your system does not have `yacc` or `bison` (the GNU equivalent), you need to install one of them first. +The default in the `makefile` is `bison`; you will have +to edit the `makefile` to use `yacc`. -NOTE: This version uses ANSI C (C 99), as you should also. We have +NOTE: This version uses ISO/IEC C99, as you should also. We have compiled this without any changes using `gcc -Wall` and/or local C compilers on a variety of systems, but new systems or compilers may raise some new complaint; reports of difficulties are welcome. This compiles without change on Macintosh OS X using `gcc` and the standard developer tools. You can also use `make CC=g++` to build with the GNU C++ compiler, should you choose to do so. -The version of `malloc` that comes with some systems is sometimes -astonishly slow. If `awk` seems slow, you might try fixing that. -More generally, turning on optimization can significantly improve -`awk`'s speed, perhaps by 1/3 for highest levels. - ## A Note About Releases -We don't do releases. +We don't usually do releases. ## A Note About Maintenance NOTICE! Maintenance of this program is on a ''best effort'' basis. We try to get to issues and pull requests as quickly as we can. Unfortunately, however, keeping this program going is not at the top of our priority list. #### Last Updated -Sat Jul 25 14:00:07 EDT 2021 +Mon 05 Feb 2024 08:46:55 IST diff --git a/contrib/one-true-awk/awk.1 b/contrib/one-true-awk/awk.1 index b3698eb07d1a..496a2a652379 100644 --- a/contrib/one-true-awk/awk.1 +++ b/contrib/one-true-awk/awk.1 @@ -1,684 +1,696 @@ .de EX .nf .ft CW .. .de EE .br .fi .ft 1 .. .de TF .IP "" "\w'\fB\\$1\ \ \fP'u" .PD 0 .. .TH AWK 1 .CT 1 files prog_other .SH NAME awk \- pattern-directed scanning and processing language .SH SYNOPSIS .B awk [ .BI \-F .I fs +| +.B \-\^\-csv ] [ .BI \-v .I var=value ] [ .I 'prog' | .BI \-f .I progfile ] [ .I file ... ] .SH DESCRIPTION .I Awk scans each input .I file for lines that match any of a set of patterns specified literally in .I prog or in one or more files specified as .B \-f .IR progfile . With each pattern there can be an associated action that will be performed when a line of a .I file matches the pattern. Each line is matched against the pattern portion of every pattern-action statement; the associated action is performed for each matched pattern. The file name .B \- means the standard input. Any .I file of the form .I var=value is treated as an assignment, not a filename, and is executed at the time it would have been opened if it were a filename. The option .B \-v followed by .I var=value is an assignment to be done before .I prog is executed; any number of .B \-v options may be present. The .B \-F .I fs option defines the input field separator to be the regular expression .IR fs . +The +.B \-\^\-csv +option causes +.I awk +to process records using (more or less) standard comma-separated values +(CSV) format. .PP An input line is normally made up of fields separated by white space, or by the regular expression .BR FS . The fields are denoted .BR $1 , .BR $2 , \&..., while .B $0 refers to the entire line. If .BR FS is null, the input line is split into one field per character. .PP A pattern-action statement has the form: .IP .IB pattern " { " action " } .PP A missing .BI { " action " } means print the line; a missing pattern always matches. Pattern-action statements are separated by newlines or semicolons. .PP An action is a sequence of statements. A statement can be one of the following: .PP .EX .ta \w'\f(CWdelete array[expression]\fR'u .RS .nf .ft CW if(\fI expression \fP)\fI statement \fP\fR[ \fPelse\fI statement \fP\fR]\fP while(\fI expression \fP)\fI statement\fP for(\fI expression \fP;\fI expression \fP;\fI expression \fP)\fI statement\fP for(\fI var \fPin\fI array \fP)\fI statement\fP do\fI statement \fPwhile(\fI expression \fP) break continue {\fR [\fP\fI statement ... \fP\fR] \fP} \fIexpression\fP #\fR commonly\fP\fI var = expression\fP print\fR [ \fP\fIexpression-list \fP\fR] \fP\fR[ \fP>\fI expression \fP\fR]\fP printf\fI format \fP\fR[ \fP,\fI expression-list \fP\fR] \fP\fR[ \fP>\fI expression \fP\fR]\fP return\fR [ \fP\fIexpression \fP\fR]\fP next #\fR skip remaining patterns on this input line\fP nextfile #\fR skip rest of this file, open next, start at top\fP delete\fI array\fP[\fI expression \fP] #\fR delete an array element\fP delete\fI array\fP #\fR delete all elements of array\fP exit\fR [ \fP\fIexpression \fP\fR]\fP #\fR exit immediately; status is \fP\fIexpression\fP .fi .RE .EE .DT .PP Statements are terminated by semicolons, newlines or right braces. An empty .I expression-list stands for .BR $0 . String constants are quoted \&\f(CW"\ "\fR, with the usual C escapes recognized within. Expressions take on string or numeric values as appropriate, and are built using the operators .B + \- * / % ^ (exponentiation), and concatenation (indicated by white space). The operators .B ! ++ \-\- += \-= *= /= %= ^= > >= < <= == != ?: are also available in expressions. Variables may be scalars, array elements (denoted .IB x [ i ] \fR) or fields. Variables are initialized to the null string. Array subscripts may be any string, not necessarily numeric; this allows for a form of associative memory. Multiple subscripts such as .B [i,j,k] are permitted; the constituents are concatenated, separated by the value of .BR SUBSEP . .PP The .B print statement prints its arguments on the standard output (or on a file if .BI > " file or .BI >> " file is present or on a pipe if .BI | " cmd is present), separated by the current output field separator, and terminated by the output record separator. .I file and .I cmd may be literal names or parenthesized expressions; identical string values in different statements denote the same open file. The .B printf statement formats its expression list according to the .I format (see .IR printf (3)). The built-in function .BI close( expr ) closes the file or pipe .IR expr . The built-in function .BI fflush( expr ) flushes any buffered output for the file or pipe .IR expr . .PP The mathematical functions .BR atan2 , .BR cos , .BR exp , .BR log , .BR sin , and .B sqrt are built in. Other built-in functions: -.TF length +.TF "\fBlength(\fR[\fIv\^\fR]\fB)\fR" .TP -.B length +\fBlength(\fR[\fIv\^\fR]\fB)\fR the length of its argument taken as a string, number of elements in an array for an array argument, or length of .B $0 if no argument. .TP -.B rand +.B rand() random number on [0,1). .TP -.B srand +\fBsrand(\fR[\fIs\^\fR]\fB)\fR sets seed for .B rand and returns the previous seed. .TP -.B int +.BI int( x\^ ) truncates to an integer value. .TP \fBsubstr(\fIs\fB, \fIm\fR [\fB, \fIn\^\fR]\fB)\fR the .IR n -character substring of .I s that begins at position .I m counted from 1. If no .IR n , use the rest of the string. .TP .BI index( s , " t" ) the position in .I s where the string .I t occurs, or 0 if it does not. .TP .BI match( s , " r" ) the position in .I s where the regular expression .I r occurs, or 0 if it does not. The variables .B RSTART and .B RLENGTH are set to the position and length of the matched string. .TP \fBsplit(\fIs\fB, \fIa \fR[\fB, \fIfs\^\fR]\fB)\fR splits the string .I s into array elements .IB a [1] \fR, .IB a [2] \fR, \&..., .IB a [ n ] \fR, and returns .IR n . The separation is done with the regular expression .I fs or with the field separator .B FS if .I fs is not given. An empty string as field separator splits the string into one array element per character. .TP \fBsub(\fIr\fB, \fIt \fR[, \fIs\^\fR]\fB) substitutes .I t for the first occurrence of the regular expression .I r in the string .IR s . If .I s is not given, .B $0 is used. .TP \fBgsub(\fIr\fB, \fIt \fR[, \fIs\^\fR]\fB) same as .B sub except that all occurrences of the regular expression are replaced; .B sub and .B gsub return the number of replacements. .TP \fBgensub(\fIpat\fB, \fIrepl\fB, \fIhow\fR [\fB, \fItarget\fR]\fB)\fR replaces instances of .I pat in .I target with .IR repl . If .I how is \fB"g"\fR or \fB"G"\fR, do so globally. Otherwise, .I how is a number indicating which occurrence to replace. If no .IR target , use .BR $0 . Return the resulting string; .I target is not modified. .TP .BI sprintf( fmt , " expr" , " ...\fB) the string resulting from formatting .I expr ... according to the .IR printf (3) format .IR fmt . .TP .B systime() returns the current date and time as a standard ``seconds since the epoch'' value. .TP .BI strftime( fmt ", " timestamp\^ ) formats .I timestamp (a value in seconds since the epoch) according to .IR fmt , which is a format string as supported by .IR strftime (3). Both .I timestamp and .I fmt may be omitted; if no .IR timestamp , the current time of day is used, and if no .IR fmt , a default format of \fB"%a %b %e %H:%M:%S %Z %Y"\fR is used. .TP .BI system( cmd ) executes .I cmd and returns its exit status. This will be \-1 upon error, .IR cmd 's exit status upon a normal exit, 256 + .I sig upon death-by-signal, where .I sig is the number of the murdering signal, or 512 + .I sig if there was a core dump. .TP .BI tolower( str ) returns a copy of .I str with all upper-case characters translated to their corresponding lower-case equivalents. .TP .BI toupper( str ) returns a copy of .I str with all lower-case characters translated to their corresponding upper-case equivalents. .PD .PP The ``function'' .B getline sets .B $0 to the next input record from the current input file; .B getline .BI < " file sets .B $0 to the next record from .IR file . .B getline .I x sets variable .I x instead. Finally, .IB cmd " | getline pipes the output of .I cmd into .BR getline ; each call of .B getline returns the next line of output from .IR cmd . In all cases, .B getline returns 1 for a successful input, 0 for end of file, and \-1 for an error. .PP The functions .BR compl , .BR and , .BR or , .BR xor , .BR lshift , and .B rshift peform the corresponding bitwise operations on their operands, which are first truncated to integer. .PP Patterns are arbitrary Boolean combinations (with .BR "! || &&" ) of regular expressions and relational expressions. Regular expressions are as in .IR egrep ; see .IR grep (1). Isolated regular expressions in a pattern apply to the entire line. Regular expressions may also occur in relational expressions, using the operators .B ~ and .BR !~ . .BI / re / is a constant regular expression; any string (constant or variable) may be used as a regular expression, except in the position of an isolated regular expression in a pattern. .PP A pattern may consist of two patterns separated by a comma; in this case, the action is performed for all lines from an occurrence of the first pattern -though an occurrence of the second. +through an occurrence of the second, inclusive. .PP A relational expression is one of the following: .IP .I expression matchop regular-expression .br .I expression relop expression .br .IB expression " in " array-name .br -.BI ( expr , expr,... ") in " array-name +.BI ( expr ,\| expr ,\| ... ") in " array-name .PP where a .I relop is any of the six relational operators in C, and a .I matchop is either .B ~ (matches) or .B !~ (does not match). A conditional is an arithmetic expression, a relational expression, or a Boolean combination of these. .PP The special patterns .B BEGIN and .B END may be used to capture control before the first input line is read and after the last. .B BEGIN and .B END do not combine with other patterns. They may appear multiple times in a program and execute in the order they are read by .IR awk . .PP Variable names with special meanings: .TF FILENAME .TP .B ARGC argument count, assignable. .TP .B ARGV argument array, assignable; non-null members are taken as filenames. .TP .B CONVFMT conversion format used when converting numbers (default .BR "%.6g" ). .TP .B ENVIRON array of environment variables; subscripts are names. .TP .B FILENAME the name of the current input file. .TP .B FNR ordinal number of the current record in the current file. .TP .B FS regular expression used to separate fields; also settable by option .BI \-F fs\fR. .TP .BR NF number of fields in the current record. .TP .B NR ordinal number of the current record. .TP .B OFMT output format for numbers (default .BR "%.6g" ). .TP .B OFS output field separator (default space). .TP .B ORS output record separator (default newline). .TP .B RLENGTH the length of a string matched by .BR match . .TP .B RS input record separator (default newline). If empty, blank lines separate records. If more than one character long, .B RS is treated as a regular expression, and records are separated by text matching the expression. .TP .B RSTART the start position of a string matched by .BR match . .TP .B SUBSEP separates multiple subscripts (default 034). .PD .PP Functions may be defined (at the position of a pattern-action statement) thus: .IP .B -function foo(a, b, c) { ...; return x } +function foo(a, b, c) { ... } .PP Parameters are passed by value if scalar and by reference if array name; functions may be called recursively. Parameters are local to the function; all other variables are global. Thus local variables may be created by providing excess parameters in the function definition. .SH ENVIRONMENT VARIABLES If .B POSIXLY_CORRECT is set in the environment, then .I awk follows the POSIX rules for .B sub and .B gsub with respect to consecutive backslashes and ampersands. .SH EXAMPLES .TP .EX length($0) > 72 .EE Print lines longer than 72 characters. .TP .EX { print $2, $1 } .EE Print first two fields in opposite order. .PP .EX BEGIN { FS = ",[ \et]*|[ \et]+" } { print $2, $1 } .EE .ns .IP Same, with input fields separated by comma and/or spaces and tabs. .PP .EX .nf { s += $1 } END { print "sum is", s, " average is", s/NR } .fi .EE .ns .IP Add up first column, print sum and average. .TP .EX /start/, /stop/ .EE Print all lines between start/stop pairs. .PP .EX .nf BEGIN { # Simulate echo(1) for (i = 1; i < ARGC; i++) printf "%s ", ARGV[i] printf "\en" exit } .fi .EE .SH SEE ALSO .IR grep (1), .IR lex (1), .IR sed (1) .br A. V. Aho, B. W. Kernighan, P. J. Weinberger, -.IR "The AWK Programming Language" , -Addison-Wesley, 1988. ISBN 0-201-07981-X. +.IR "The AWK Programming Language, Second Edition" , +Addison-Wesley, 2024. ISBN 978-0-13-826972-2, 0-13-826972-6. .SH BUGS There are no explicit conversions between numbers and strings. To force an expression to be treated as a number add 0 to it; to force it to be treated as a string concatenate \&\f(CW""\fP to it. .PP The scope rules for variables in functions are a botch; the syntax is worse. .PP -Only eight-bit characters sets are handled correctly. +Input is expected to be UTF-8 encoded. Other multibyte +character sets are not handled. +However, in eight-bit locales, +.I awk +treats each input byte as a separate character. .SH UNUSUAL FLOATING-POINT VALUES .I Awk was designed before IEEE 754 arithmetic defined Not-A-Number (NaN) and Infinity values, which are supported by all modern floating-point hardware. .PP Because .I awk uses .IR strtod (3) and .IR atof (3) to convert string values to double-precision floating-point values, modern C libraries also convert strings starting with .B inf and .B nan into infinity and NaN values respectively. This led to strange results, with something like this: .PP .EX .nf echo nancy | awk '{ print $1 + 0 }' .fi .EE .PP printing .B nan instead of zero. .PP .I Awk now follows GNU AWK, and prefilters string values before attempting to convert them to numbers, as follows: .TP .I "Hexadecimal values" Hexadecimal values (allowed since C99) convert to zero, as they did prior to C99. .TP .I "NaN values" The two strings .B +nan and .B \-nan (case independent) convert to NaN. No others do. (NaNs can have signs.) .TP .I "Infinity values" The two strings .B +inf and .B \-inf (case independent) convert to positive and negative infinity, respectively. No others do. diff --git a/contrib/one-true-awk/awk.h b/contrib/one-true-awk/awk.h index 51c00df9f279..740447ee2167 100644 --- a/contrib/one-true-awk/awk.h +++ b/contrib/one-true-awk/awk.h @@ -1,266 +1,283 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ #include #include #include #if __STDC_VERSION__ <= 199901L #define noreturn #else #include #endif typedef double Awkfloat; /* unsigned char is more trouble than it's worth */ typedef unsigned char uschar; -#define xfree(a) { if ((a) != NULL) { free((void *)(intptr_t)(a)); (a) = NULL; } } +#define xfree(a) { free((void *)(intptr_t)(a)); (a) = NULL; } /* * We sometimes cheat writing read-only pointers to NUL-terminate them * and then put back the original value */ #define setptr(ptr, a) (*(char *)(intptr_t)(ptr)) = (a) #define NN(p) ((p) ? (p) : "(null)") /* guaranteed non-null for DPRINTF */ #define DEBUG #ifdef DEBUG # define DPRINTF(...) if (dbg) printf(__VA_ARGS__) #else # define DPRINTF(...) #endif extern enum compile_states { RUNNING, COMPILING, ERROR_PRINTING } compile_time; extern bool safe; /* false => unsafe, true => safe */ #define RECSIZE (8 * 1024) /* sets limit on records, fields, etc., etc. */ extern int recsize; /* size of current record, orig RECSIZE */ +extern size_t awk_mb_cur_max; /* max size of a multi-byte character */ + extern char EMPTY[]; /* this avoid -Wwritable-strings issues */ extern char **FS; extern char **RS; extern char **ORS; extern char **OFS; extern char **OFMT; extern Awkfloat *NR; extern Awkfloat *FNR; extern Awkfloat *NF; extern char **FILENAME; extern char **SUBSEP; extern Awkfloat *RSTART; extern Awkfloat *RLENGTH; +extern bool CSV; /* true for csv input */ + extern char *record; /* points to $0 */ extern int lineno; /* line number in awk program */ extern int errorflag; /* 1 if error has occurred */ extern bool donefld; /* true if record broken into fields */ extern bool donerec; /* true if record is valid (no fld has changed */ extern int dbg; extern const char *patbeg; /* beginning of pattern matched */ extern int patlen; /* length of pattern matched. set in b.c */ /* Cell: all information about a variable or constant */ typedef struct Cell { uschar ctype; /* OCELL, OBOOL, OJUMP, etc. */ uschar csub; /* CCON, CTEMP, CFLD, etc. */ char *nval; /* name, for variables only */ char *sval; /* string value */ Awkfloat fval; /* value as number */ int tval; /* type info: STR|NUM|ARR|FCN|FLD|CON|DONTFREE|CONVC|CONVO */ char *fmt; /* CONVFMT/OFMT value used to convert from number */ struct Cell *cnext; /* ptr to next if chained */ } Cell; typedef struct Array { /* symbol table array */ int nelem; /* elements in table right now */ int size; /* size of tab */ Cell **tab; /* hash table pointers */ } Array; #define NSYMTAB 50 /* initial size of a symbol table */ extern Array *symtab; extern Cell *nrloc; /* NR */ extern Cell *fnrloc; /* FNR */ extern Cell *fsloc; /* FS */ extern Cell *nfloc; /* NF */ extern Cell *ofsloc; /* OFS */ extern Cell *orsloc; /* ORS */ extern Cell *rsloc; /* RS */ extern Cell *rstartloc; /* RSTART */ extern Cell *rlengthloc; /* RLENGTH */ extern Cell *subseploc; /* SUBSEP */ extern Cell *symtabloc; /* SYMTAB */ /* Cell.tval values: */ #define NUM 01 /* number value is valid */ #define STR 02 /* string value is valid */ #define DONTFREE 04 /* string space is not freeable */ #define CON 010 /* this is a constant */ #define ARR 020 /* this is an array */ #define FCN 040 /* this is a function name */ #define FLD 0100 /* this is a field $1, $2, ... */ #define REC 0200 /* this is $0 */ #define CONVC 0400 /* string was converted from number via CONVFMT */ #define CONVO 01000 /* string was converted from number via OFMT */ /* function types */ #define FLENGTH 1 #define FSQRT 2 #define FEXP 3 #define FLOG 4 #define FINT 5 #define FSYSTEM 6 #define FRAND 7 #define FSRAND 8 #define FSIN 9 #define FCOS 10 #define FATAN 11 #define FTOUPPER 12 #define FTOLOWER 13 #define FFLUSH 14 #define FAND 15 #define FFOR 16 #define FXOR 17 #define FCOMPL 18 #define FLSHIFT 19 #define FRSHIFT 20 #define FSYSTIME 21 #define FSTRFTIME 22 /* Node: parse tree is made of nodes, with Cell's at bottom */ typedef struct Node { int ntype; struct Node *nnext; int lineno; int nobj; struct Node *narg[1]; /* variable: actual size set by calling malloc */ } Node; #define NIL ((Node *) 0) extern Node *winner; extern Node *nullstat; extern Node *nullnode; /* ctypes */ #define OCELL 1 #define OBOOL 2 #define OJUMP 3 /* Cell subtypes: csub */ #define CFREE 7 #define CCOPY 6 #define CCON 5 #define CTEMP 4 #define CNAME 3 #define CVAR 2 #define CFLD 1 #define CUNK 0 /* bool subtypes */ #define BTRUE 11 #define BFALSE 12 /* jump subtypes */ #define JEXIT 21 #define JNEXT 22 #define JBREAK 23 #define JCONT 24 #define JRET 25 #define JNEXTFILE 26 /* node types */ #define NVALUE 1 #define NSTAT 2 #define NEXPR 3 extern int pairstack[], paircnt; #define notlegal(n) (n <= FIRSTTOKEN || n >= LASTTOKEN || proctab[n-FIRSTTOKEN] == nullproc) #define isvalue(n) ((n)->ntype == NVALUE) #define isexpr(n) ((n)->ntype == NEXPR) #define isjump(n) ((n)->ctype == OJUMP) #define isexit(n) ((n)->csub == JEXIT) #define isbreak(n) ((n)->csub == JBREAK) #define iscont(n) ((n)->csub == JCONT) #define isnext(n) ((n)->csub == JNEXT || (n)->csub == JNEXTFILE) #define isret(n) ((n)->csub == JRET) #define isrec(n) ((n)->tval & REC) #define isfld(n) ((n)->tval & FLD) #define isstr(n) ((n)->tval & STR) #define isnum(n) ((n)->tval & NUM) #define isarr(n) ((n)->tval & ARR) #define isfcn(n) ((n)->tval & FCN) #define istrue(n) ((n)->csub == BTRUE) #define istemp(n) ((n)->csub == CTEMP) #define isargument(n) ((n)->nobj == ARG) /* #define freeable(p) (!((p)->tval & DONTFREE)) */ #define freeable(p) ( ((p)->tval & (STR|DONTFREE)) == STR ) /* structures used by regular expression matching machinery, mostly b.c: */ -#define NCHARS (256+3) /* 256 handles 8-bit chars; 128 does 7-bit */ +#define NCHARS (1256+3) /* 256 handles 8-bit chars; 128 does 7-bit */ + /* BUG: some overflows (caught) if we use 256 */ /* watch out in match(), etc. */ #define HAT (NCHARS+2) /* matches ^ in regular expr */ #define NSTATES 32 typedef struct rrow { long ltype; /* long avoids pointer warnings on 64-bit */ union { int i; Node *np; uschar *up; + int *rp; /* rune representation of char class */ } lval; /* because Al stores a pointer in it! */ int *lfollow; } rrow; +typedef struct gtte { /* gototab entry */ + unsigned int ch; + unsigned int state; +} gtte; + +typedef struct gtt { /* gototab */ + size_t allocated; + size_t inuse; + gtte *entries; +} gtt; + typedef struct fa { - unsigned int **gototab; + gtt *gototab; uschar *out; uschar *restr; int **posns; int state_count; bool anchor; int use; int initstat; int curstat; int accept; struct rrow re[1]; /* variable: actual size set by calling malloc */ } fa; #include "proto.h" diff --git a/contrib/one-true-awk/awkgram.y b/contrib/one-true-awk/awkgram.y index f37073d1f9ac..233253a4307b 100644 --- a/contrib/one-true-awk/awkgram.y +++ b/contrib/one-true-awk/awkgram.y @@ -1,508 +1,515 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ %{ #include #include #include "awk.h" void checkdup(Node *list, Cell *item); int yywrap(void) { return(1); } Node *beginloc = 0; Node *endloc = 0; bool infunc = false; /* = true if in arglist or body of func */ int inloop = 0; /* >= 1 if in while, for, do; can't be bool, since loops can next */ char *curfname = 0; /* current function name */ Node *arglist = 0; /* list of args for current function */ %} %union { Node *p; Cell *cp; int i; char *s; } %token FIRSTTOKEN /* must be first */ %token

PROGRAM PASTAT PASTAT2 XBEGIN XEND %token NL ',' '{' '(' '|' ';' '/' ')' '}' '[' ']' %token ARRAY %token MATCH NOTMATCH MATCHOP %token FINAL DOT ALL CCL NCCL CHAR OR STAR QUEST PLUS EMPTYRE ZERO %token AND BOR APPEND EQ GE GT LE LT NE IN %token ARG BLTIN BREAK CLOSE CONTINUE DELETE DO EXIT FOR FUNC %token GENSUB SUB GSUB IF INDEX LSUBSTR MATCHFCN NEXT NEXTFILE %token ADD MINUS MULT DIVIDE MOD %token ASSIGN ASGNOP ADDEQ SUBEQ MULTEQ DIVEQ MODEQ POWEQ %token PRINT PRINTF SPRINTF %token

ELSE INTEST CONDEXPR %token POSTINCR PREINCR POSTDECR PREDECR %token VAR IVAR VARNF CALL NUMBER STRING %token REGEXPR %type

pas pattern ppattern plist pplist patlist prarg term re %type

pa_pat pa_stat pa_stats %type reg_expr %type

simple_stmt opt_simple_stmt stmt stmtlist %type

var varname funcname varlist %type

for if else while %type do st %type pst opt_pst lbrace rbrace rparen comma nl opt_nl and bor %type subop print %type string %right ASGNOP %right '?' %right ':' %left BOR %left AND %left GETLINE %nonassoc APPEND EQ GE GT LE LT NE MATCHOP IN '|' %left ARG BLTIN BREAK CALL CLOSE CONTINUE DELETE DO EXIT FOR FUNC %left GSUB IF INDEX LSUBSTR MATCHFCN NEXT NUMBER %left PRINT PRINTF RETURN SPLIT SPRINTF STRING SUB SUBSTR %left REGEXPR VAR VARNF IVAR WHILE '(' %left CAT %left '+' '-' %left '*' '/' '%' %left NOT UMINUS UPLUS %right POWER %right DECR INCR %left INDIRECT %token LASTTOKEN /* must be last */ %% program: pas { if (errorflag==0) winner = (Node *)stat3(PROGRAM, beginloc, $1, endloc); } | error { yyclearin; bracecheck(); SYNTAX("bailing out"); } ; and: AND | and NL ; bor: BOR | bor NL ; comma: ',' | comma NL ; do: DO | do NL ; else: ELSE | else NL ; for: FOR '(' opt_simple_stmt ';' opt_nl pattern ';' opt_nl opt_simple_stmt rparen {inloop++;} stmt { --inloop; $$ = stat4(FOR, $3, notnull($6), $9, $12); } | FOR '(' opt_simple_stmt ';' ';' opt_nl opt_simple_stmt rparen {inloop++;} stmt { --inloop; $$ = stat4(FOR, $3, NIL, $7, $10); } | FOR '(' varname IN varname rparen {inloop++;} stmt { --inloop; $$ = stat3(IN, $3, makearr($5), $8); } ; funcname: VAR { setfname($1); } | CALL { setfname($1); } ; if: IF '(' pattern rparen { $$ = notnull($3); } ; lbrace: '{' | lbrace NL ; nl: NL | nl NL ; opt_nl: /* empty */ { $$ = 0; } | nl ; opt_pst: /* empty */ { $$ = 0; } | pst ; opt_simple_stmt: /* empty */ { $$ = 0; } | simple_stmt ; pas: opt_pst { $$ = 0; } | opt_pst pa_stats opt_pst { $$ = $2; } ; pa_pat: pattern { $$ = notnull($1); } ; pa_stat: pa_pat { $$ = stat2(PASTAT, $1, stat2(PRINT, rectonode(), NIL)); } | pa_pat lbrace stmtlist '}' { $$ = stat2(PASTAT, $1, $3); } | pa_pat ',' opt_nl pa_pat { $$ = pa2stat($1, $4, stat2(PRINT, rectonode(), NIL)); } | pa_pat ',' opt_nl pa_pat lbrace stmtlist '}' { $$ = pa2stat($1, $4, $6); } | lbrace stmtlist '}' { $$ = stat2(PASTAT, NIL, $2); } | XBEGIN lbrace stmtlist '}' { beginloc = linkum(beginloc, $3); $$ = 0; } | XEND lbrace stmtlist '}' { endloc = linkum(endloc, $3); $$ = 0; } | FUNC funcname '(' varlist rparen {infunc = true;} lbrace stmtlist '}' { infunc = false; curfname=0; defn((Cell *)$2, $4, $8); $$ = 0; } ; pa_stats: pa_stat | pa_stats opt_pst pa_stat { $$ = linkum($1, $3); } ; patlist: pattern | patlist comma pattern { $$ = linkum($1, $3); } ; ppattern: var ASGNOP ppattern { $$ = op2($2, $1, $3); } | ppattern '?' ppattern ':' ppattern %prec '?' { $$ = op3(CONDEXPR, notnull($1), $3, $5); } | ppattern bor ppattern %prec BOR { $$ = op2(BOR, notnull($1), notnull($3)); } | ppattern and ppattern %prec AND { $$ = op2(AND, notnull($1), notnull($3)); } - | ppattern MATCHOP reg_expr { $$ = op3($2, NIL, $1, (Node*)makedfa($3, 0)); } + | ppattern MATCHOP reg_expr { $$ = op3($2, NIL, $1, (Node*)makedfa($3, 0)); free($3); } | ppattern MATCHOP ppattern - { if (constnode($3)) + { if (constnode($3)) { $$ = op3($2, NIL, $1, (Node*)makedfa(strnode($3), 0)); - else + free($3); + } else $$ = op3($2, (Node *)1, $1, $3); } | ppattern IN varname { $$ = op2(INTEST, $1, makearr($3)); } | '(' plist ')' IN varname { $$ = op2(INTEST, $2, makearr($5)); } | ppattern term %prec CAT { $$ = op2(CAT, $1, $2); } | re | term ; pattern: var ASGNOP pattern { $$ = op2($2, $1, $3); } | pattern '?' pattern ':' pattern %prec '?' { $$ = op3(CONDEXPR, notnull($1), $3, $5); } | pattern bor pattern %prec BOR { $$ = op2(BOR, notnull($1), notnull($3)); } | pattern and pattern %prec AND { $$ = op2(AND, notnull($1), notnull($3)); } | pattern EQ pattern { $$ = op2($2, $1, $3); } | pattern GE pattern { $$ = op2($2, $1, $3); } | pattern GT pattern { $$ = op2($2, $1, $3); } | pattern LE pattern { $$ = op2($2, $1, $3); } | pattern LT pattern { $$ = op2($2, $1, $3); } | pattern NE pattern { $$ = op2($2, $1, $3); } - | pattern MATCHOP reg_expr { $$ = op3($2, NIL, $1, (Node*)makedfa($3, 0)); } + | pattern MATCHOP reg_expr { $$ = op3($2, NIL, $1, (Node*)makedfa($3, 0)); free($3); } | pattern MATCHOP pattern - { if (constnode($3)) + { if (constnode($3)) { $$ = op3($2, NIL, $1, (Node*)makedfa(strnode($3), 0)); - else + free($3); + } else $$ = op3($2, (Node *)1, $1, $3); } | pattern IN varname { $$ = op2(INTEST, $1, makearr($3)); } | '(' plist ')' IN varname { $$ = op2(INTEST, $2, makearr($5)); } | pattern '|' GETLINE var { if (safe) SYNTAX("cmd | getline is unsafe"); else $$ = op3(GETLINE, $4, itonp($2), $1); } | pattern '|' GETLINE { if (safe) SYNTAX("cmd | getline is unsafe"); else $$ = op3(GETLINE, (Node*)0, itonp($2), $1); } | pattern term %prec CAT { $$ = op2(CAT, $1, $2); } | re | term ; plist: pattern comma pattern { $$ = linkum($1, $3); } | plist comma pattern { $$ = linkum($1, $3); } ; pplist: ppattern | pplist comma ppattern { $$ = linkum($1, $3); } ; prarg: /* empty */ { $$ = rectonode(); } | pplist | '(' plist ')' { $$ = $2; } ; print: PRINT | PRINTF ; pst: NL | ';' | pst NL | pst ';' ; rbrace: '}' | rbrace NL ; re: reg_expr - { $$ = op3(MATCH, NIL, rectonode(), (Node*)makedfa($1, 0)); } + { $$ = op3(MATCH, NIL, rectonode(), (Node*)makedfa($1, 0)); free($1); } | NOT re { $$ = op1(NOT, notnull($2)); } ; reg_expr: '/' {startreg();} REGEXPR '/' { $$ = $3; } ; rparen: ')' | rparen NL ; simple_stmt: print prarg '|' term { if (safe) SYNTAX("print | is unsafe"); else $$ = stat3($1, $2, itonp($3), $4); } | print prarg APPEND term { if (safe) SYNTAX("print >> is unsafe"); else $$ = stat3($1, $2, itonp($3), $4); } | print prarg GT term { if (safe) SYNTAX("print > is unsafe"); else $$ = stat3($1, $2, itonp($3), $4); } | print prarg { $$ = stat3($1, $2, NIL, NIL); } | DELETE varname '[' patlist ']' { $$ = stat2(DELETE, makearr($2), $4); } | DELETE varname { $$ = stat2(DELETE, makearr($2), 0); } | pattern { $$ = exptostat($1); } | error { yyclearin; SYNTAX("illegal statement"); } ; st: nl | ';' opt_nl ; stmt: BREAK st { if (!inloop) SYNTAX("break illegal outside of loops"); $$ = stat1(BREAK, NIL); } | CONTINUE st { if (!inloop) SYNTAX("continue illegal outside of loops"); $$ = stat1(CONTINUE, NIL); } | do {inloop++;} stmt {--inloop;} WHILE '(' pattern ')' st { $$ = stat2(DO, $3, notnull($7)); } | EXIT pattern st { $$ = stat1(EXIT, $2); } | EXIT st { $$ = stat1(EXIT, NIL); } | for | if stmt else stmt { $$ = stat3(IF, $1, $2, $4); } | if stmt { $$ = stat3(IF, $1, $2, NIL); } | lbrace stmtlist rbrace { $$ = $2; } | NEXT st { if (infunc) SYNTAX("next is illegal inside a function"); $$ = stat1(NEXT, NIL); } | NEXTFILE st { if (infunc) SYNTAX("nextfile is illegal inside a function"); $$ = stat1(NEXTFILE, NIL); } | RETURN pattern st { $$ = stat1(RETURN, $2); } | RETURN st { $$ = stat1(RETURN, NIL); } | simple_stmt st | while {inloop++;} stmt { --inloop; $$ = stat2(WHILE, $1, $3); } | ';' opt_nl { $$ = 0; } ; stmtlist: stmt | stmtlist stmt { $$ = linkum($1, $2); } ; subop: SUB | GSUB ; string: STRING | string STRING { $$ = catstr($1, $2); } ; term: term '/' ASGNOP term { $$ = op2(DIVEQ, $1, $4); } | term '+' term { $$ = op2(ADD, $1, $3); } | term '-' term { $$ = op2(MINUS, $1, $3); } | term '*' term { $$ = op2(MULT, $1, $3); } | term '/' term { $$ = op2(DIVIDE, $1, $3); } | term '%' term { $$ = op2(MOD, $1, $3); } | term POWER term { $$ = op2(POWER, $1, $3); } | '-' term %prec UMINUS { $$ = op1(UMINUS, $2); } | '+' term %prec UMINUS { $$ = op1(UPLUS, $2); } | NOT term %prec UMINUS { $$ = op1(NOT, notnull($2)); } | BLTIN '(' ')' { $$ = op2(BLTIN, itonp($1), rectonode()); } | BLTIN '(' patlist ')' { $$ = op2(BLTIN, itonp($1), $3); } | BLTIN { $$ = op2(BLTIN, itonp($1), rectonode()); } | CALL '(' ')' { $$ = op2(CALL, celltonode($1,CVAR), NIL); } | CALL '(' patlist ')' { $$ = op2(CALL, celltonode($1,CVAR), $3); } | CLOSE term { $$ = op1(CLOSE, $2); } | DECR var { $$ = op1(PREDECR, $2); } | INCR var { $$ = op1(PREINCR, $2); } | var DECR { $$ = op1(POSTDECR, $1); } | var INCR { $$ = op1(POSTINCR, $1); } | GENSUB '(' reg_expr comma pattern comma pattern ')' { $$ = op5(GENSUB, NIL, (Node*)makedfa($3, 1), $5, $7, rectonode()); } | GENSUB '(' pattern comma pattern comma pattern ')' - { if (constnode($3)) + { if (constnode($3)) { $$ = op5(GENSUB, NIL, (Node *)makedfa(strnode($3), 1), $5, $7, rectonode()); - else + free($3); + } else $$ = op5(GENSUB, (Node *)1, $3, $5, $7, rectonode()); } | GENSUB '(' reg_expr comma pattern comma pattern comma pattern ')' { $$ = op5(GENSUB, NIL, (Node*)makedfa($3, 1), $5, $7, $9); } | GENSUB '(' pattern comma pattern comma pattern comma pattern ')' - { if (constnode($3)) + { if (constnode($3)) { $$ = op5(GENSUB, NIL, (Node *)makedfa(strnode($3),1), $5,$7,$9); - else + free($3); + } else $$ = op5(GENSUB, (Node *)1, $3, $5, $7, $9); } | GETLINE var LT term { $$ = op3(GETLINE, $2, itonp($3), $4); } | GETLINE LT term { $$ = op3(GETLINE, NIL, itonp($2), $3); } | GETLINE var { $$ = op3(GETLINE, $2, NIL, NIL); } | GETLINE { $$ = op3(GETLINE, NIL, NIL, NIL); } | INDEX '(' pattern comma pattern ')' { $$ = op2(INDEX, $3, $5); } | INDEX '(' pattern comma reg_expr ')' { SYNTAX("index() doesn't permit regular expressions"); $$ = op2(INDEX, $3, (Node*)$5); } | '(' pattern ')' { $$ = $2; } | MATCHFCN '(' pattern comma reg_expr ')' - { $$ = op3(MATCHFCN, NIL, $3, (Node*)makedfa($5, 1)); } + { $$ = op3(MATCHFCN, NIL, $3, (Node*)makedfa($5, 1)); free($5); } | MATCHFCN '(' pattern comma pattern ')' - { if (constnode($5)) + { if (constnode($5)) { $$ = op3(MATCHFCN, NIL, $3, (Node*)makedfa(strnode($5), 1)); - else + free($5); + } else $$ = op3(MATCHFCN, (Node *)1, $3, $5); } | NUMBER { $$ = celltonode($1, CCON); } | SPLIT '(' pattern comma varname comma pattern ')' /* string */ { $$ = op4(SPLIT, $3, makearr($5), $7, (Node*)STRING); } | SPLIT '(' pattern comma varname comma reg_expr ')' /* const /regexp/ */ - { $$ = op4(SPLIT, $3, makearr($5), (Node*)makedfa($7, 1), (Node *)REGEXPR); } + { $$ = op4(SPLIT, $3, makearr($5), (Node*)makedfa($7, 1), (Node *)REGEXPR); free($7); } | SPLIT '(' pattern comma varname ')' { $$ = op4(SPLIT, $3, makearr($5), NIL, (Node*)STRING); } /* default */ | SPRINTF '(' patlist ')' { $$ = op1($1, $3); } | string { $$ = celltonode($1, CCON); } | subop '(' reg_expr comma pattern ')' - { $$ = op4($1, NIL, (Node*)makedfa($3, 1), $5, rectonode()); } + { $$ = op4($1, NIL, (Node*)makedfa($3, 1), $5, rectonode()); free($3); } | subop '(' pattern comma pattern ')' - { if (constnode($3)) + { if (constnode($3)) { $$ = op4($1, NIL, (Node*)makedfa(strnode($3), 1), $5, rectonode()); - else + free($3); + } else $$ = op4($1, (Node *)1, $3, $5, rectonode()); } | subop '(' reg_expr comma pattern comma var ')' - { $$ = op4($1, NIL, (Node*)makedfa($3, 1), $5, $7); } + { $$ = op4($1, NIL, (Node*)makedfa($3, 1), $5, $7); free($3); } | subop '(' pattern comma pattern comma var ')' - { if (constnode($3)) + { if (constnode($3)) { $$ = op4($1, NIL, (Node*)makedfa(strnode($3), 1), $5, $7); - else + free($3); + } else $$ = op4($1, (Node *)1, $3, $5, $7); } | SUBSTR '(' pattern comma pattern comma pattern ')' { $$ = op3(SUBSTR, $3, $5, $7); } | SUBSTR '(' pattern comma pattern ')' { $$ = op3(SUBSTR, $3, $5, NIL); } | var ; var: varname | varname '[' patlist ']' { $$ = op2(ARRAY, makearr($1), $3); } | IVAR { $$ = op1(INDIRECT, celltonode($1, CVAR)); } | INDIRECT term { $$ = op1(INDIRECT, $2); } ; varlist: /* nothing */ { arglist = $$ = 0; } | VAR { arglist = $$ = celltonode($1,CVAR); } | varlist comma VAR { checkdup($1, $3); arglist = $$ = linkum($1,celltonode($3,CVAR)); } ; varname: VAR { $$ = celltonode($1, CVAR); } | ARG { $$ = op1(ARG, itonp($1)); } | VARNF { $$ = op1(VARNF, (Node *) $1); } ; while: WHILE '(' pattern rparen { $$ = notnull($3); } ; %% void setfname(Cell *p) { if (isarr(p)) SYNTAX("%s is an array, not a function", p->nval); else if (isfcn(p)) SYNTAX("you can't define function %s more than once", p->nval); curfname = p->nval; } int constnode(Node *p) { return isvalue(p) && ((Cell *) (p->narg[0]))->csub == CCON; } char *strnode(Node *p) { return ((Cell *)(p->narg[0]))->sval; } Node *notnull(Node *n) { switch (n->nobj) { case LE: case LT: case EQ: case NE: case GT: case GE: case BOR: case AND: case NOT: return n; default: return op2(NE, n, nullnode); } } void checkdup(Node *vl, Cell *cp) /* check if name already in list */ { char *s = cp->nval; for ( ; vl; vl = vl->nnext) { if (strcmp(s, ((Cell *)(vl->narg[0]))->nval) == 0) { SYNTAX("duplicate argument %s", s); break; } } } diff --git a/contrib/one-true-awk/b.c b/contrib/one-true-awk/b.c index 804236656e22..4c438fab4cd4 100644 --- a/contrib/one-true-awk/b.c +++ b/contrib/one-true-awk/b.c @@ -1,1341 +1,1562 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ /* lasciate ogne speranza, voi ch'intrate. */ #define DEBUG #include #include #include #include #include #include "awk.h" #include "awkgram.tab.h" #define MAXLIN 22 #define type(v) (v)->nobj /* badly overloaded here */ #define info(v) (v)->ntype /* badly overloaded here */ #define left(v) (v)->narg[0] #define right(v) (v)->narg[1] #define parent(v) (v)->nnext #define LEAF case CCL: case NCCL: case CHAR: case DOT: case FINAL: case ALL: #define ELEAF case EMPTYRE: /* empty string in regexp */ #define UNARY case STAR: case PLUS: case QUEST: /* encoding in tree Nodes: leaf (CCL, NCCL, CHAR, DOT, FINAL, ALL, EMPTYRE): left is index, right contains value or pointer to value unary (STAR, PLUS, QUEST): left is child, right is null binary (CAT, OR): left and right are children parent contains pointer to parent */ int *setvec; int *tmpset; int maxsetvec = 0; int rtok; /* next token in current re */ int rlxval; static const uschar *rlxstr; static const uschar *prestr; /* current position in current re */ static const uschar *lastre; /* origin of last re */ static const uschar *lastatom; /* origin of last Atom */ static const uschar *starttok; static const uschar *basestr; /* starts with original, replaced during repetition processing */ static const uschar *firstbasestr; static int setcnt; static int poscnt; const char *patbeg; int patlen; #define NFA 128 /* cache this many dynamic fa's */ fa *fatab[NFA]; int nfatab = 0; /* entries in fatab */ +extern int u8_nextlen(const char *s); + + +/* utf-8 mechanism: + + For most of Awk, utf-8 strings just "work", since they look like + null-terminated sequences of 8-bit bytes. + + Functions like length(), index(), and substr() have to operate + in units of utf-8 characters. The u8_* functions in run.c + handle this. + + Regular expressions are more complicated, since the basic + mechanism of the goto table used 8-bit byte indices into the + gototab entries to compute the next state. Unicode is a lot + bigger, so the gototab entries are now structs with a character + and a next state. These are sorted by code point and binary + searched. + + Throughout the RE mechanism in b.c, utf-8 characters are + converted to their utf-32 value. This mostly shows up in + cclenter, which expands character class ranges like a-z and now + alpha-omega. The size of a gototab array is still about 256. + This should be dynamic, but for now things work ok for a single + code page of Unicode, which is the most likely case. + + The code changes are localized in run.c and b.c. I have added a + handful of functions to somewhat better hide the implementation, + but a lot more could be done. + + */ + +static int entry_cmp(const void *l, const void *r); +static int get_gototab(fa*, int, int); +static int set_gototab(fa*, int, int, int); +static void clear_gototab(fa*, int); +extern int u8_rune(int *, const char *); + static int * intalloc(size_t n, const char *f) { int *p = (int *) calloc(n, sizeof(int)); if (p == NULL) overflo(f); return p; } static void resizesetvec(const char *f) { if (maxsetvec == 0) maxsetvec = MAXLIN; else maxsetvec *= 4; setvec = (int *) realloc(setvec, maxsetvec * sizeof(*setvec)); tmpset = (int *) realloc(tmpset, maxsetvec * sizeof(*tmpset)); if (setvec == NULL || tmpset == NULL) overflo(f); } static void resize_state(fa *f, int state) { - unsigned int **p; + gtt *p; uschar *p2; int **p3; int i, new_count; if (++state < f->state_count) return; new_count = state + 10; /* needs to be tuned */ - p = (unsigned int **) realloc(f->gototab, new_count * sizeof(f->gototab[0])); + p = (gtt *) realloc(f->gototab, new_count * sizeof(gtt)); if (p == NULL) goto out; f->gototab = p; p2 = (uschar *) realloc(f->out, new_count * sizeof(f->out[0])); if (p2 == NULL) goto out; f->out = p2; p3 = (int **) realloc(f->posns, new_count * sizeof(f->posns[0])); if (p3 == NULL) goto out; f->posns = p3; for (i = f->state_count; i < new_count; ++i) { - f->gototab[i] = (unsigned int *) calloc(NCHARS, sizeof(**f->gototab)); - if (f->gototab[i] == NULL) + f->gototab[i].entries = (gtte *) calloc(NCHARS, sizeof(gtte)); + if (f->gototab[i].entries == NULL) goto out; - f->out[i] = 0; + f->gototab[i].allocated = NCHARS; + f->gototab[i].inuse = 0; + f->out[i] = 0; f->posns[i] = NULL; } f->state_count = new_count; return; out: overflo(__func__); } fa *makedfa(const char *s, bool anchor) /* returns dfa for reg expr s */ { int i, use, nuse; fa *pfa; static int now = 1; if (setvec == NULL) { /* first time through any RE */ resizesetvec(__func__); } if (compile_time != RUNNING) /* a constant for sure */ return mkdfa(s, anchor); for (i = 0; i < nfatab; i++) /* is it there already? */ if (fatab[i]->anchor == anchor && strcmp((const char *) fatab[i]->restr, s) == 0) { fatab[i]->use = now++; return fatab[i]; } pfa = mkdfa(s, anchor); if (nfatab < NFA) { /* room for another */ fatab[nfatab] = pfa; fatab[nfatab]->use = now++; nfatab++; return pfa; } use = fatab[0]->use; /* replace least-recently used */ nuse = 0; for (i = 1; i < nfatab; i++) if (fatab[i]->use < use) { use = fatab[i]->use; nuse = i; } freefa(fatab[nuse]); fatab[nuse] = pfa; pfa->use = now++; return pfa; } fa *mkdfa(const char *s, bool anchor) /* does the real work of making a dfa */ /* anchor = true for anchored matches, else false */ { Node *p, *p1; fa *f; firstbasestr = (const uschar *) s; basestr = firstbasestr; p = reparse(s); p1 = op2(CAT, op2(STAR, op2(ALL, NIL, NIL), NIL), p); /* put ALL STAR in front of reg. exp. */ p1 = op2(CAT, p1, op2(FINAL, NIL, NIL)); /* put FINAL after reg. exp. */ poscnt = 0; penter(p1); /* enter parent pointers and leaf indices */ if ((f = (fa *) calloc(1, sizeof(fa) + poscnt * sizeof(rrow))) == NULL) overflo(__func__); f->accept = poscnt-1; /* penter has computed number of positions in re */ cfoll(f, p1); /* set up follow sets */ freetr(p1); resize_state(f, 1); f->posns[0] = intalloc(*(f->re[0].lfollow), __func__); f->posns[1] = intalloc(1, __func__); *f->posns[1] = 0; f->initstat = makeinit(f, anchor); f->anchor = anchor; f->restr = (uschar *) tostring(s); if (firstbasestr != basestr) { if (basestr) xfree(basestr); } return f; } int makeinit(fa *f, bool anchor) { int i, k; f->curstat = 2; f->out[2] = 0; k = *(f->re[0].lfollow); xfree(f->posns[2]); f->posns[2] = intalloc(k + 1, __func__); for (i = 0; i <= k; i++) { (f->posns[2])[i] = (f->re[0].lfollow)[i]; } if ((f->posns[2])[1] == f->accept) f->out[2] = 1; - for (i = 0; i < NCHARS; i++) - f->gototab[2][i] = 0; + clear_gototab(f, 2); f->curstat = cgoto(f, 2, HAT); if (anchor) { *f->posns[2] = k-1; /* leave out position 0 */ for (i = 0; i < k; i++) { (f->posns[0])[i] = (f->posns[2])[i]; } f->out[0] = f->out[2]; if (f->curstat != 2) --(*f->posns[f->curstat]); } return f->curstat; } void penter(Node *p) /* set up parent pointers and leaf indices */ { switch (type(p)) { ELEAF LEAF info(p) = poscnt; poscnt++; break; UNARY penter(left(p)); parent(left(p)) = p; break; case CAT: case OR: penter(left(p)); penter(right(p)); parent(left(p)) = p; parent(right(p)) = p; break; case ZERO: break; default: /* can't happen */ FATAL("can't happen: unknown type %d in penter", type(p)); break; } } void freetr(Node *p) /* free parse tree */ { switch (type(p)) { ELEAF LEAF xfree(p); break; UNARY case ZERO: freetr(left(p)); xfree(p); break; case CAT: case OR: freetr(left(p)); freetr(right(p)); xfree(p); break; default: /* can't happen */ FATAL("can't happen: unknown type %d in freetr", type(p)); break; } } /* in the parsing of regular expressions, metacharacters like . have */ /* to be seen literally; \056 is not a metacharacter. */ -int hexstr(const uschar **pp) /* find and eval hex string at pp, return new p */ +int hexstr(const uschar **pp, int max) /* find and eval hex string at pp, return new p */ { /* only pick up one 8-bit byte (2 chars) */ const uschar *p; int n = 0; int i; - for (i = 0, p = *pp; i < 2 && isxdigit(*p); i++, p++) { - if (isdigit(*p)) + for (i = 0, p = *pp; i < max && isxdigit(*p); i++, p++) { + if (isdigit((int) *p)) n = 16 * n + *p - '0'; else if (*p >= 'a' && *p <= 'f') n = 16 * n + *p - 'a' + 10; else if (*p >= 'A' && *p <= 'F') n = 16 * n + *p - 'A' + 10; } *pp = p; return n; } + + #define isoctdigit(c) ((c) >= '0' && (c) <= '7') /* multiple use of arg */ int quoted(const uschar **pp) /* pick up next thing after a \\ */ /* and increment *pp */ { const uschar *p = *pp; int c; - if ((c = *p++) == 't') +/* BUG: should advance by utf-8 char even if makes no sense */ + + if ((c = *p++) == 't') { c = '\t'; - else if (c == 'n') + } else if (c == 'n') { c = '\n'; - else if (c == 'f') + } else if (c == 'f') { c = '\f'; - else if (c == 'r') + } else if (c == 'r') { c = '\r'; - else if (c == 'b') + } else if (c == 'b') { c = '\b'; - else if (c == 'v') + } else if (c == 'v') { c = '\v'; - else if (c == 'a') + } else if (c == 'a') { c = '\a'; - else if (c == '\\') + } else if (c == '\\') { c = '\\'; - else if (c == 'x') { /* hexadecimal goo follows */ - c = hexstr(&p); /* this adds a null if number is invalid */ + } else if (c == 'x') { /* 2 hex digits follow */ + c = hexstr(&p, 2); /* this adds a null if number is invalid */ + } else if (c == 'u') { /* unicode char number up to 8 hex digits */ + c = hexstr(&p, 8); } else if (isoctdigit(c)) { /* \d \dd \ddd */ int n = c - '0'; if (isoctdigit(*p)) { n = 8 * n + *p++ - '0'; if (isoctdigit(*p)) n = 8 * n + *p++ - '0'; } c = n; } /* else */ /* c = c; */ *pp = p; return c; } -char *cclenter(const char *argp) /* add a character class */ +int *cclenter(const char *argp) /* add a character class */ { int i, c, c2; - const uschar *op, *p = (const uschar *) argp; - uschar *bp; - static uschar *buf = NULL; + int n; + const uschar *p = (const uschar *) argp; + int *bp, *retp; + static int *buf = NULL; static int bufsz = 100; - op = p; - if (buf == NULL && (buf = (uschar *) malloc(bufsz)) == NULL) + if (buf == NULL && (buf = (int *) calloc(bufsz, sizeof(int))) == NULL) FATAL("out of space for character class [%.10s...] 1", p); bp = buf; - for (i = 0; (c = *p++) != 0; ) { + for (i = 0; *p != 0; ) { + n = u8_rune(&c, (const char *) p); + p += n; if (c == '\\') { c = quoted(&p); } else if (c == '-' && i > 0 && bp[-1] != 0) { if (*p != 0) { c = bp[-1]; - c2 = *p++; + /* c2 = *p++; */ + n = u8_rune(&c2, (const char *) p); + p += n; if (c2 == '\\') - c2 = quoted(&p); + c2 = quoted(&p); /* BUG: sets p, has to be u8 size */ if (c > c2) { /* empty; ignore */ bp--; i--; continue; } while (c < c2) { - if (!adjbuf((char **) &buf, &bufsz, bp-buf+2, 100, (char **) &bp, "cclenter1")) - FATAL("out of space for character class [%.10s...] 2", p); + if (i >= bufsz) { + bufsz *= 2; + buf = (int *) realloc(buf, bufsz * sizeof(int)); + if (buf == NULL) + FATAL("out of space for character class [%.10s...] 2", p); + bp = buf + i; + } *bp++ = ++c; i++; } continue; } } - if (!adjbuf((char **) &buf, &bufsz, bp-buf+2, 100, (char **) &bp, "cclenter2")) - FATAL("out of space for character class [%.10s...] 3", p); + if (i >= bufsz) { + bufsz *= 2; + buf = (int *) realloc(buf, bufsz * sizeof(int)); + if (buf == NULL) + FATAL("out of space for character class [%.10s...] 2", p); + bp = buf + i; + } *bp++ = c; i++; } *bp = 0; - DPRINTF("cclenter: in = |%s|, out = |%s|\n", op, buf); - xfree(op); - return (char *) tostring((char *) buf); + /* DPRINTF("cclenter: in = |%s|, out = |%s|\n", op, buf); BUG: can't print array of int */ + /* xfree(op); BUG: what are we freeing here? */ + retp = (int *) calloc(bp-buf+1, sizeof(int)); + for (i = 0; i < bp-buf+1; i++) + retp[i] = buf[i]; + return retp; } void overflo(const char *s) { FATAL("regular expression too big: out of space in %.30s...", s); } void cfoll(fa *f, Node *v) /* enter follow set of each leaf of vertex v into lfollow[leaf] */ { int i; int *p; switch (type(v)) { ELEAF LEAF f->re[info(v)].ltype = type(v); f->re[info(v)].lval.np = right(v); while (f->accept >= maxsetvec) { /* guessing here! */ resizesetvec(__func__); } for (i = 0; i <= f->accept; i++) setvec[i] = 0; setcnt = 0; follow(v); /* computes setvec and setcnt */ p = intalloc(setcnt + 1, __func__); f->re[info(v)].lfollow = p; *p = setcnt; for (i = f->accept; i >= 0; i--) if (setvec[i] == 1) *++p = i; break; UNARY cfoll(f,left(v)); break; case CAT: case OR: cfoll(f,left(v)); cfoll(f,right(v)); break; case ZERO: break; default: /* can't happen */ FATAL("can't happen: unknown type %d in cfoll", type(v)); } } int first(Node *p) /* collects initially active leaves of p into setvec */ /* returns 0 if p matches empty string */ { int b, lp; switch (type(p)) { ELEAF LEAF lp = info(p); /* look for high-water mark of subscripts */ while (setcnt >= maxsetvec || lp >= maxsetvec) { /* guessing here! */ resizesetvec(__func__); } if (type(p) == EMPTYRE) { setvec[lp] = 0; return(0); } if (setvec[lp] != 1) { setvec[lp] = 1; setcnt++; } - if (type(p) == CCL && (*(char *) right(p)) == '\0') + if (type(p) == CCL && (*(int *) right(p)) == 0) return(0); /* empty CCL */ return(1); case PLUS: if (first(left(p)) == 0) return(0); return(1); case STAR: case QUEST: first(left(p)); return(0); case CAT: if (first(left(p)) == 0 && first(right(p)) == 0) return(0); return(1); case OR: b = first(right(p)); if (first(left(p)) == 0 || b == 0) return(0); return(1); case ZERO: return 0; } FATAL("can't happen: unknown type %d in first", type(p)); /* can't happen */ return(-1); } void follow(Node *v) /* collects leaves that can follow v into setvec */ { Node *p; if (type(v) == FINAL) return; p = parent(v); switch (type(p)) { case STAR: case PLUS: first(v); follow(p); return; case OR: case QUEST: follow(p); return; case CAT: if (v == left(p)) { /* v is left child of p */ if (first(right(p)) == 0) { follow(p); return; } } else /* v is right child */ follow(p); return; } } -int member(int c, const char *sarg) /* is c in s? */ +int member(int c, int *sarg) /* is c in s? */ { - const uschar *s = (const uschar *) sarg; + int *s = (int *) sarg; while (*s) if (c == *s++) return(1); return(0); } +static void resize_gototab(fa *f, int state) +{ + size_t new_size = f->gototab[state].allocated * 2; + gtte *p = (gtte *) realloc(f->gototab[state].entries, new_size * sizeof(gtte)); + if (p == NULL) + overflo(__func__); + + // need to initialized the new memory to zero + size_t orig_size = f->gototab[state].allocated; // 2nd half of new mem is this size + memset(p + orig_size, 0, orig_size * sizeof(gtte)); // clean it out + + f->gototab[state].allocated = new_size; // update gototab info + f->gototab[state].entries = p; +} + +static int get_gototab(fa *f, int state, int ch) /* hide gototab implementation */ +{ + gtte key; + gtte *item; + + key.ch = ch; + key.state = 0; /* irrelevant */ + item = (gtte *) bsearch(& key, f->gototab[state].entries, + f->gototab[state].inuse, sizeof(gtte), + entry_cmp); + + if (item == NULL) + return 0; + else + return item->state; +} + +static int entry_cmp(const void *l, const void *r) +{ + const gtte *left, *right; + + left = (const gtte *) l; + right = (const gtte *) r; + + return left->ch - right->ch; +} + +static int set_gototab(fa *f, int state, int ch, int val) /* hide gototab implementation */ +{ + if (f->gototab[state].inuse == 0) { + f->gototab[state].entries[0].ch = ch; + f->gototab[state].entries[0].state = val; + f->gototab[state].inuse++; + return val; + } else if (ch > f->gototab[state].entries[f->gototab[state].inuse-1].ch) { + // not seen yet, insert and return + gtt *tab = & f->gototab[state]; + if (tab->inuse + 1 >= tab->allocated) + resize_gototab(f, state); + + f->gototab[state].entries[f->gototab[state].inuse-1].ch = ch; + f->gototab[state].entries[f->gototab[state].inuse-1].state = val; + f->gototab[state].inuse++; + return val; + } else { + // maybe we have it, maybe we don't + gtte key; + gtte *item; + + key.ch = ch; + key.state = 0; /* irrelevant */ + item = (gtte *) bsearch(& key, f->gototab[state].entries, + f->gototab[state].inuse, sizeof(gtte), + entry_cmp); + + if (item != NULL) { + // we have it, update state and return + item->state = val; + return item->state; + } + // otherwise, fall through to insert and reallocate. + } + + gtt *tab = & f->gototab[state]; + if (tab->inuse + 1 >= tab->allocated) + resize_gototab(f, state); + ++tab->inuse; + f->gototab[state].entries[tab->inuse].ch = ch; + f->gototab[state].entries[tab->inuse].state = val; + + qsort(f->gototab[state].entries, + f->gototab[state].inuse, sizeof(gtte), entry_cmp); + + return val; /* not used anywhere at the moment */ +} + +static void clear_gototab(fa *f, int state) +{ + memset(f->gototab[state].entries, 0, + f->gototab[state].allocated * sizeof(gtte)); + f->gototab[state].inuse = 0; +} + int match(fa *f, const char *p0) /* shortest match ? */ { int s, ns; + int n; + int rune; const uschar *p = (const uschar *) p0; + /* return pmatch(f, p0); does it matter whether longest or shortest? */ + s = f->initstat; assert (s < f->state_count); if (f->out[s]) return(1); do { /* assert(*p < NCHARS); */ - if ((ns = f->gototab[s][*p]) != 0) + n = u8_rune(&rune, (const char *) p); + if ((ns = get_gototab(f, s, rune)) != 0) s = ns; else - s = cgoto(f, s, *p); + s = cgoto(f, s, rune); if (f->out[s]) return(1); - } while (*p++ != 0); + if (*p == 0) + break; + p += n; + } while (1); /* was *p++ != 0 */ return(0); } int pmatch(fa *f, const char *p0) /* longest match, for sub */ { int s, ns; + int n; + int rune; const uschar *p = (const uschar *) p0; const uschar *q; s = f->initstat; assert(s < f->state_count); patbeg = (const char *)p; patlen = -1; do { q = p; do { if (f->out[s]) /* final state */ patlen = q-p; /* assert(*q < NCHARS); */ - if ((ns = f->gototab[s][*q]) != 0) + n = u8_rune(&rune, (const char *) q); + if ((ns = get_gototab(f, s, rune)) != 0) s = ns; else - s = cgoto(f, s, *q); + s = cgoto(f, s, rune); assert(s < f->state_count); if (s == 1) { /* no transition */ if (patlen >= 0) { patbeg = (const char *) p; return(1); } else goto nextin; /* no match */ } - } while (*q++ != 0); + if (*q == 0) + break; + q += n; + } while (1); + q++; /* was *q++ */ if (f->out[s]) patlen = q-p-1; /* don't count $ */ if (patlen >= 0) { patbeg = (const char *) p; return(1); } nextin: s = 2; - } while (*p++); + if (*p == 0) + break; + n = u8_rune(&rune, (const char *) p); + p += n; + } while (1); /* was *p++ */ return (0); } int nematch(fa *f, const char *p0) /* non-empty match, for sub */ { int s, ns; + int n; + int rune; const uschar *p = (const uschar *) p0; const uschar *q; s = f->initstat; assert(s < f->state_count); patbeg = (const char *)p; patlen = -1; while (*p) { q = p; do { if (f->out[s]) /* final state */ patlen = q-p; /* assert(*q < NCHARS); */ - if ((ns = f->gototab[s][*q]) != 0) + n = u8_rune(&rune, (const char *) q); + if ((ns = get_gototab(f, s, rune)) != 0) s = ns; else - s = cgoto(f, s, *q); + s = cgoto(f, s, rune); if (s == 1) { /* no transition */ if (patlen > 0) { patbeg = (const char *) p; return(1); } else goto nnextin; /* no nonempty match */ } - } while (*q++ != 0); + if (*q == 0) + break; + q += n; + } while (1); + q++; if (f->out[s]) patlen = q-p-1; /* don't count $ */ if (patlen > 0 ) { patbeg = (const char *) p; return(1); } nnextin: s = 2; p++; } return (0); } +#define MAX_UTF_BYTES 4 // UTF-8 is up to 4 bytes long + /* * NAME * fnematch * * DESCRIPTION * A stream-fed version of nematch which transfers characters to a * null-terminated buffer. All characters up to and including the last * character of the matching text or EOF are placed in the buffer. If * a match is found, patbeg and patlen are set appropriately. * * RETURN VALUES * false No match found. * true Match found. */ bool fnematch(fa *pfa, FILE *f, char **pbuf, int *pbufsize, int quantum) { - char *buf = *pbuf; + char *i, *j, *k, *buf = *pbuf; int bufsize = *pbufsize; - int c, i, j, k, ns, s; + int c, n, ns, s; s = pfa->initstat; patlen = 0; /* - * All indices relative to buf. - * i <= j <= k <= bufsize + * buf <= i <= j <= k <= buf+bufsize * * i: origin of active substring * j: current character - * k: destination of next getc() + * k: destination of the next getc */ - i = -1, k = 0; - do { - j = i++; - do { - if (++j == k) { - if (k == bufsize) - if (!adjbuf((char **) &buf, &bufsize, bufsize+1, quantum, 0, "fnematch")) - FATAL("stream '%.30s...' too long", buf); - buf[k++] = (c = getc(f)) != EOF ? c : 0; - } - c = (uschar)buf[j]; - /* assert(c < NCHARS); */ - if ((ns = pfa->gototab[s][c]) != 0) - s = ns; - else - s = cgoto(pfa, s, c); + i = j = k = buf; - if (pfa->out[s]) { /* final state */ - patlen = j - i + 1; - if (c == 0) /* don't count $ */ - patlen--; + do { + /* + * Call u8_rune with at least MAX_UTF_BYTES ahead in + * the buffer until EOF interferes. + */ + if (k - j < MAX_UTF_BYTES) { + if (k + MAX_UTF_BYTES > buf + bufsize) { + adjbuf((char **) &buf, &bufsize, + bufsize + MAX_UTF_BYTES, + quantum, 0, "fnematch"); + } + for (n = MAX_UTF_BYTES ; n > 0; n--) { + *k++ = (c = getc(f)) != EOF ? c : 0; + if (c == EOF) { + if (ferror(f)) + FATAL("fnematch: getc error"); + break; + } } - } while (buf[j] && s != 1); + } + + j += u8_rune(&c, j); + + if ((ns = get_gototab(pfa, s, c)) != 0) + s = ns; + else + s = cgoto(pfa, s, c); + + if (pfa->out[s]) { /* final state */ + patbeg = i; + patlen = j - i; + if (c == 0) /* don't count $ */ + patlen--; + } + + if (c && s != 1) + continue; /* origin i still viable, next j */ + if (patlen) + break; /* best match found */ + + /* no match at origin i, next i and start over */ + i += u8_rune(&c, i); + if (c == 0) + break; /* no match */ + j = i; s = 2; - } while (buf[i] && !patlen); + } while (1); /* adjbuf() may have relocated a resized buffer. Inform the world. */ *pbuf = buf; *pbufsize = bufsize; if (patlen) { - patbeg = (char *) buf + i; /* * Under no circumstances is the last character fed to * the automaton part of the match. It is EOF's nullbyte, * or it sent the automaton into a state with no further * transitions available (s==1), or both. Room for a * terminating nullbyte is guaranteed. * * ungetc any chars after the end of matching text * (except for EOF's nullbyte, if present) and null * terminate the buffer. */ do - if (buf[--k] && ungetc(buf[k], f) == EOF) - FATAL("unable to ungetc '%c'", buf[k]); - while (k > i + patlen); - buf[k] = '\0'; + if (*--k && ungetc(*k, f) == EOF) + FATAL("unable to ungetc '%c'", *k); + while (k > patbeg + patlen); + *k = '\0'; return true; } else return false; } Node *reparse(const char *p) /* parses regular expression pointed to by p */ { /* uses relex() to scan regular expression */ Node *np; DPRINTF("reparse <%s>\n", p); lastre = prestr = (const uschar *) p; /* prestr points to string to be parsed */ rtok = relex(); /* GNU compatibility: an empty regexp matches anything */ if (rtok == '\0') { /* FATAL("empty regular expression"); previous */ return(op2(EMPTYRE, NIL, NIL)); } np = regexp(); if (rtok != '\0') FATAL("syntax error in regular expression %s at %s", lastre, prestr); return(np); } Node *regexp(void) /* top-level parse of reg expr */ { return (alt(concat(primary()))); } Node *primary(void) { Node *np; int savelastatom; switch (rtok) { case CHAR: lastatom = starttok; np = op2(CHAR, NIL, itonp(rlxval)); rtok = relex(); return (unary(np)); case ALL: rtok = relex(); return (unary(op2(ALL, NIL, NIL))); case EMPTYRE: rtok = relex(); return (unary(op2(EMPTYRE, NIL, NIL))); case DOT: lastatom = starttok; rtok = relex(); return (unary(op2(DOT, NIL, NIL))); case CCL: np = op2(CCL, NIL, (Node*) cclenter((const char *) rlxstr)); lastatom = starttok; rtok = relex(); return (unary(np)); case NCCL: np = op2(NCCL, NIL, (Node *) cclenter((const char *) rlxstr)); lastatom = starttok; rtok = relex(); return (unary(np)); case '^': rtok = relex(); return (unary(op2(CHAR, NIL, itonp(HAT)))); case '$': rtok = relex(); return (unary(op2(CHAR, NIL, NIL))); case '(': lastatom = starttok; savelastatom = starttok - basestr; /* Retain over recursion */ rtok = relex(); if (rtok == ')') { /* special pleading for () */ rtok = relex(); - return unary(op2(CCL, NIL, (Node *) tostring(""))); + return unary(op2(CCL, NIL, (Node *) cclenter(""))); } np = regexp(); if (rtok == ')') { lastatom = basestr + savelastatom; /* Restore */ rtok = relex(); return (unary(np)); } else FATAL("syntax error in regular expression %s at %s", lastre, prestr); default: FATAL("illegal primary in regular expression %s at %s", lastre, prestr); } return 0; /*NOTREACHED*/ } Node *concat(Node *np) { switch (rtok) { case CHAR: case DOT: case ALL: case CCL: case NCCL: case '$': case '(': return (concat(op2(CAT, np, primary()))); case EMPTYRE: rtok = relex(); - return (concat(op2(CAT, op2(CCL, NIL, (Node *) tostring("")), + return (concat(op2(CAT, op2(CCL, NIL, (Node *) cclenter("")), primary()))); } return (np); } Node *alt(Node *np) { if (rtok == OR) { rtok = relex(); return (alt(op2(OR, np, concat(primary())))); } return (np); } Node *unary(Node *np) { switch (rtok) { case STAR: rtok = relex(); return (unary(op2(STAR, np, NIL))); case PLUS: rtok = relex(); return (unary(op2(PLUS, np, NIL))); case QUEST: rtok = relex(); return (unary(op2(QUEST, np, NIL))); case ZERO: rtok = relex(); return (unary(op2(ZERO, np, NIL))); default: return (np); } } /* * Character class definitions conformant to the POSIX locale as * defined in IEEE P1003.1 draft 7 of June 2001, assuming the source * and operating character sets are both ASCII (ISO646) or supersets * thereof. * * Note that to avoid overflowing the temporary buffer used in * relex(), the expanded character class (prior to range expansion) * must be less than twice the size of their full name. */ /* Because isblank doesn't show up in any of the header files on any * system i use, it's defined here. if some other locale has a richer * definition of "blank", define HAS_ISBLANK and provide your own * version. * the parentheses here are an attempt to find a path through the maze * of macro definition and/or function and/or version provided. thanks * to nelson beebe for the suggestion; let's see if it works everywhere. */ /* #define HAS_ISBLANK */ #ifndef HAS_ISBLANK int (xisblank)(int c) { return c==' ' || c=='\t'; } #endif static const struct charclass { const char *cc_name; int cc_namelen; int (*cc_func)(int); } charclasses[] = { { "alnum", 5, isalnum }, { "alpha", 5, isalpha }, #ifndef HAS_ISBLANK { "blank", 5, xisblank }, #else { "blank", 5, isblank }, #endif { "cntrl", 5, iscntrl }, { "digit", 5, isdigit }, { "graph", 5, isgraph }, { "lower", 5, islower }, { "print", 5, isprint }, { "punct", 5, ispunct }, { "space", 5, isspace }, { "upper", 5, isupper }, { "xdigit", 6, isxdigit }, { NULL, 0, NULL }, }; #define REPEAT_SIMPLE 0 #define REPEAT_PLUS_APPENDED 1 #define REPEAT_WITH_Q 2 #define REPEAT_ZERO 3 static int replace_repeat(const uschar *reptok, int reptoklen, const uschar *atom, int atomlen, int firstnum, int secondnum, int special_case) { int i, j; uschar *buf = 0; int ret = 1; int init_q = (firstnum == 0); /* first added char will be ? */ int n_q_reps = secondnum-firstnum; /* m>n, so reduce until {1,m-n} left */ int prefix_length = reptok - basestr; /* prefix includes first rep */ int suffix_length = strlen((const char *) reptok) - reptoklen; /* string after rep specifier */ int size = prefix_length + suffix_length; if (firstnum > 1) { /* add room for reps 2 through firstnum */ size += atomlen*(firstnum-1); } /* Adjust size of buffer for special cases */ if (special_case == REPEAT_PLUS_APPENDED) { size++; /* for the final + */ } else if (special_case == REPEAT_WITH_Q) { size += init_q + (atomlen+1)* (n_q_reps-init_q); } else if (special_case == REPEAT_ZERO) { size += 2; /* just a null ERE: () */ } if ((buf = (uschar *) malloc(size + 1)) == NULL) FATAL("out of space in reg expr %.10s..", lastre); memcpy(buf, basestr, prefix_length); /* copy prefix */ j = prefix_length; if (special_case == REPEAT_ZERO) { j -= atomlen; buf[j++] = '('; buf[j++] = ')'; } for (i = 1; i < firstnum; i++) { /* copy x reps */ memcpy(&buf[j], atom, atomlen); j += atomlen; } if (special_case == REPEAT_PLUS_APPENDED) { buf[j++] = '+'; } else if (special_case == REPEAT_WITH_Q) { if (init_q) buf[j++] = '?'; for (i = init_q; i < n_q_reps; i++) { /* copy x? reps */ memcpy(&buf[j], atom, atomlen); j += atomlen; buf[j++] = '?'; } } memcpy(&buf[j], reptok+reptoklen, suffix_length); j += suffix_length; buf[j] = '\0'; /* free old basestr */ if (firstbasestr != basestr) { if (basestr) xfree(basestr); } basestr = buf; prestr = buf + prefix_length; if (special_case == REPEAT_ZERO) { prestr -= atomlen; ret++; } return ret; } static int repeat(const uschar *reptok, int reptoklen, const uschar *atom, int atomlen, int firstnum, int secondnum) { /* In general, the repetition specifier or "bound" is replaced here by an equivalent ERE string, repeating the immediately previous atom and appending ? and + as needed. Note that the first copy of the atom is left in place, except in the special_case of a zero-repeat (i.e., {0}). */ if (secondnum < 0) { /* means {n,} -> repeat n-1 times followed by PLUS */ if (firstnum < 2) { /* 0 or 1: should be handled before you get here */ FATAL("internal error"); } else { return replace_repeat(reptok, reptoklen, atom, atomlen, firstnum, secondnum, REPEAT_PLUS_APPENDED); } } else if (firstnum == secondnum) { /* {n} or {n,n} -> simply repeat n-1 times */ if (firstnum == 0) { /* {0} or {0,0} */ /* This case is unusual because the resulting replacement string might actually be SMALLER than the original ERE */ return replace_repeat(reptok, reptoklen, atom, atomlen, firstnum, secondnum, REPEAT_ZERO); } else { /* (firstnum >= 1) */ return replace_repeat(reptok, reptoklen, atom, atomlen, firstnum, secondnum, REPEAT_SIMPLE); } } else if (firstnum < secondnum) { /* {n,m} -> repeat n-1 times then alternate */ /* x{n,m} => xx...x{1, m-n+1} => xx...x?x?x?..x? */ return replace_repeat(reptok, reptoklen, atom, atomlen, firstnum, secondnum, REPEAT_WITH_Q); } else { /* Error - shouldn't be here (n>m) */ FATAL("internal error"); } return 0; } int relex(void) /* lexical analyzer for reparse */ { int c, n; int cflag; static uschar *buf = NULL; static int bufsz = 100; uschar *bp; const struct charclass *cc; int i; int num, m; bool commafound, digitfound; const uschar *startreptok; static int parens = 0; rescan: starttok = prestr; + if ((n = u8_rune(&rlxval, (const char *) prestr)) > 1) { + prestr += n; + starttok = prestr; + return CHAR; + } + switch (c = *prestr++) { case '|': return OR; case '*': return STAR; case '+': return PLUS; case '?': return QUEST; case '.': return DOT; case '\0': prestr--; return '\0'; case '^': case '$': return c; case '(': parens++; return c; case ')': if (parens) { parens--; return c; } /* unmatched close parenthesis; per POSIX, treat as literal */ rlxval = c; return CHAR; case '\\': rlxval = quoted(&prestr); return CHAR; default: rlxval = c; return CHAR; case '[': if (buf == NULL && (buf = (uschar *) malloc(bufsz)) == NULL) FATAL("out of space in reg expr %.10s..", lastre); bp = buf; if (*prestr == '^') { cflag = 1; prestr++; } else cflag = 0; - n = 2 * strlen((const char *) prestr)+1; + n = 5 * strlen((const char *) prestr)+1; /* BUG: was 2. what value? */ if (!adjbuf((char **) &buf, &bufsz, n, n, (char **) &bp, "relex1")) FATAL("out of space for reg expr %.10s...", lastre); for (; ; ) { + if ((n = u8_rune(&rlxval, (const char *) prestr)) > 1) { + for (i = 0; i < n; i++) + *bp++ = *prestr++; + continue; + } if ((c = *prestr++) == '\\') { *bp++ = '\\'; if ((c = *prestr++) == '\0') FATAL("nonterminated character class %.20s...", lastre); *bp++ = c; /* } else if (c == '\n') { */ /* FATAL("newline in character class %.20s...", lastre); */ } else if (c == '[' && *prestr == ':') { /* POSIX char class names, Dag-Erling Smorgrav, des@ofug.org */ for (cc = charclasses; cc->cc_name; cc++) if (strncmp((const char *) prestr + 1, (const char *) cc->cc_name, cc->cc_namelen) == 0) break; if (cc->cc_name != NULL && prestr[1 + cc->cc_namelen] == ':' && prestr[2 + cc->cc_namelen] == ']') { prestr += cc->cc_namelen + 3; /* * BUG: We begin at 1, instead of 0, since we * would otherwise prematurely terminate the * string for classes like [[:cntrl:]]. This * means that we can't match the NUL character, * not without first adapting the entire * program to track each string's length. */ for (i = 1; i <= UCHAR_MAX; i++) { - if (!adjbuf((char **) &buf, &bufsz, bp-buf+1, 100, (char **) &bp, "relex2")) + if (!adjbuf((char **) &buf, &bufsz, bp-buf+2, 100, (char **) &bp, "relex2")) FATAL("out of space for reg expr %.10s...", lastre); if (cc->cc_func(i)) { /* escape backslash */ if (i == '\\') { *bp++ = '\\'; n++; } *bp++ = i; n++; } } } else *bp++ = c; } else if (c == '[' && *prestr == '.') { char collate_char; prestr++; collate_char = *prestr++; if (*prestr == '.' && prestr[1] == ']') { prestr += 2; /* Found it: map via locale TBD: for now, simply return this char. This is sufficient to pass conformance test awk.ex 156 */ if (*prestr == ']') { prestr++; rlxval = collate_char; return CHAR; } } } else if (c == '[' && *prestr == '=') { char equiv_char; prestr++; equiv_char = *prestr++; if (*prestr == '=' && prestr[1] == ']') { prestr += 2; /* Found it: map via locale TBD: for now simply return this char. This is sufficient to pass conformance test awk.ex 156 */ if (*prestr == ']') { prestr++; rlxval = equiv_char; return CHAR; } } } else if (c == '\0') { FATAL("nonterminated character class %.20s", lastre); } else if (bp == buf) { /* 1st char is special */ *bp++ = c; } else if (c == ']') { *bp++ = 0; rlxstr = (uschar *) tostring((char *) buf); if (cflag == 0) return CCL; else return NCCL; } else *bp++ = c; } break; case '{': - if (isdigit(*(prestr))) { + if (isdigit((int) *(prestr))) { num = 0; /* Process as a repetition */ n = -1; m = -1; commafound = false; digitfound = false; startreptok = prestr-1; /* Remember start of previous atom here ? */ } else { /* just a { char, not a repetition */ rlxval = c; return CHAR; } for (; ; ) { if ((c = *prestr++) == '}') { if (commafound) { if (digitfound) { /* {n,m} */ m = num; if (m < n) FATAL("illegal repetition expression: class %.20s", lastre); if (n == 0 && m == 1) { return QUEST; } } else { /* {n,} */ if (n == 0) return STAR; else if (n == 1) return PLUS; } } else { if (digitfound) { /* {n} same as {n,n} */ n = num; m = num; } else { /* {} */ FATAL("illegal repetition expression: class %.20s", lastre); } } if (repeat(starttok, prestr-starttok, lastatom, startreptok - lastatom, n, m) > 0) { if (n == 0 && m == 0) { return ZERO; } /* must rescan input for next token */ goto rescan; } /* Failed to replace: eat up {...} characters and treat like just PLUS */ return PLUS; } else if (c == '\0') { FATAL("nonterminated character class %.20s", lastre); } else if (isdigit(c)) { num = 10 * num + c - '0'; digitfound = true; } else if (c == ',') { if (commafound) FATAL("illegal repetition expression: class %.20s", lastre); /* looking for {n,} or {n,m} */ commafound = true; n = num; digitfound = false; /* reset */ num = 0; } else { FATAL("illegal repetition expression: class %.20s", lastre); } } break; } } int cgoto(fa *f, int s, int c) { int *p, *q; int i, j, k; - assert(c == HAT || c < NCHARS); + /* assert(c == HAT || c < NCHARS); BUG: seg fault if disable test */ while (f->accept >= maxsetvec) { /* guessing here! */ resizesetvec(__func__); } for (i = 0; i <= f->accept; i++) setvec[i] = 0; setcnt = 0; resize_state(f, s); /* compute positions of gototab[s,c] into setvec */ p = f->posns[s]; for (i = 1; i <= *p; i++) { if ((k = f->re[p[i]].ltype) != FINAL) { if ((k == CHAR && c == ptoi(f->re[p[i]].lval.np)) || (k == DOT && c != 0 && c != HAT) || (k == ALL && c != 0) || (k == EMPTYRE && c != 0) - || (k == CCL && member(c, (char *) f->re[p[i]].lval.up)) - || (k == NCCL && !member(c, (char *) f->re[p[i]].lval.up) && c != 0 && c != HAT)) { + || (k == CCL && member(c, (int *) f->re[p[i]].lval.rp)) + || (k == NCCL && !member(c, (int *) f->re[p[i]].lval.rp) && c != 0 && c != HAT)) { q = f->re[p[i]].lfollow; for (j = 1; j <= *q; j++) { if (q[j] >= maxsetvec) { resizesetvec(__func__); } if (setvec[q[j]] == 0) { setcnt++; setvec[q[j]] = 1; } } } } } /* determine if setvec is a previous state */ tmpset[0] = setcnt; j = 1; for (i = f->accept; i >= 0; i--) if (setvec[i]) { tmpset[j++] = i; } resize_state(f, f->curstat > s ? f->curstat : s); /* tmpset == previous state? */ for (i = 1; i <= f->curstat; i++) { p = f->posns[i]; if ((k = tmpset[0]) != p[0]) goto different; for (j = 1; j <= k; j++) if (tmpset[j] != p[j]) goto different; /* setvec is state i */ if (c != HAT) - f->gototab[s][c] = i; + set_gototab(f, s, c, i); return i; different:; } /* add tmpset to current set of states */ ++(f->curstat); resize_state(f, f->curstat); - for (i = 0; i < NCHARS; i++) - f->gototab[f->curstat][i] = 0; + clear_gototab(f, f->curstat); xfree(f->posns[f->curstat]); p = intalloc(setcnt + 1, __func__); f->posns[f->curstat] = p; if (c != HAT) - f->gototab[s][c] = f->curstat; + set_gototab(f, s, c, f->curstat); for (i = 0; i <= setcnt; i++) p[i] = tmpset[i]; if (setvec[f->accept]) f->out[f->curstat] = 1; else f->out[f->curstat] = 0; return f->curstat; } void freefa(fa *f) /* free a finite automaton */ { int i; if (f == NULL) return; for (i = 0; i < f->state_count; i++) - xfree(f->gototab[i]) + xfree(f->gototab[i].entries); + xfree(f->gototab); for (i = 0; i <= f->curstat; i++) xfree(f->posns[i]); for (i = 0; i <= f->accept; i++) { xfree(f->re[i].lfollow); if (f->re[i].ltype == CCL || f->re[i].ltype == NCCL) xfree(f->re[i].lval.np); } xfree(f->restr); xfree(f->out); xfree(f->posns); xfree(f->gototab); xfree(f); } diff --git a/contrib/one-true-awk/bugs-fixed/REGRESS b/contrib/one-true-awk/bugs-fixed/REGRESS index 07160031ca07..acdbeebb6271 100755 --- a/contrib/one-true-awk/bugs-fixed/REGRESS +++ b/contrib/one-true-awk/bugs-fixed/REGRESS @@ -1,28 +1,32 @@ -#! /bin/bash +#! /bin/sh if [ ! -f ../a.out ] then echo Making executable (cd .. ; make) || exit 0 fi for i in *.awk do echo === $i OUT=${i%.awk}.OUT OK=${i%.awk}.ok + OK2=${i%.awk}.ok2 IN=${i%.awk}.in input= if [ -f $IN ] then input=$IN fi ../a.out -f $i $input > $OUT 2>&1 if cmp -s $OK $OUT + then + rm -f $OUT + elif [ -f $OK2 ] && cmp -s $OK2 $OUT then rm -f $OUT else - echo ++++ $i failed! + echo '++++ $i failed!' fi done diff --git a/contrib/one-true-awk/bugs-fixed/getline-corruption.awk b/contrib/one-true-awk/bugs-fixed/getline-corruption.awk new file mode 100644 index 000000000000..461e551cfff5 --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/getline-corruption.awk @@ -0,0 +1,5 @@ +BEGIN { + getline l + getline l + print (s=substr(l,1,10)) " len=" length(s) +} diff --git a/contrib/one-true-awk/bugs-fixed/getline-corruption.in b/contrib/one-true-awk/bugs-fixed/getline-corruption.in new file mode 100644 index 000000000000..78981922613b --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/getline-corruption.in @@ -0,0 +1 @@ +a diff --git a/contrib/one-true-awk/bugs-fixed/getline-corruption.ok b/contrib/one-true-awk/bugs-fixed/getline-corruption.ok new file mode 100644 index 000000000000..3efb54597c6d --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/getline-corruption.ok @@ -0,0 +1 @@ +a len=1 diff --git a/contrib/one-true-awk/bugs-fixed/matchop-deref.awk b/contrib/one-true-awk/bugs-fixed/matchop-deref.awk new file mode 100644 index 000000000000..6c066aad911d --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/matchop-deref.awk @@ -0,0 +1,11 @@ +function foo() { + return "aaaaaab" +} + +BEGIN { + print match(foo(), "b") +} + +{ + print match(substr($0, 1), "b") +} diff --git a/contrib/one-true-awk/bugs-fixed/matchop-deref.bad b/contrib/one-true-awk/bugs-fixed/matchop-deref.bad new file mode 100644 index 000000000000..343ee5c2f6cb --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/matchop-deref.bad @@ -0,0 +1,2 @@ +-1 +-1 diff --git a/contrib/one-true-awk/bugs-fixed/matchop-deref.in b/contrib/one-true-awk/bugs-fixed/matchop-deref.in new file mode 100644 index 000000000000..0d197e1b6a30 --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/matchop-deref.in @@ -0,0 +1 @@ +aaaaaab diff --git a/contrib/one-true-awk/bugs-fixed/matchop-deref.ok b/contrib/one-true-awk/bugs-fixed/matchop-deref.ok new file mode 100644 index 000000000000..49019db80789 --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/matchop-deref.ok @@ -0,0 +1,2 @@ +7 +7 diff --git a/contrib/one-true-awk/bugs-fixed/missing-precision.ok b/contrib/one-true-awk/bugs-fixed/missing-precision.ok new file mode 100644 index 000000000000..75e1e3d00446 --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/missing-precision.ok @@ -0,0 +1,2 @@ +../a.out: not enough args in printf(%*s) + source line number 1 diff --git a/contrib/one-true-awk/bugs-fixed/negative-nf.ok b/contrib/one-true-awk/bugs-fixed/negative-nf.ok new file mode 100644 index 000000000000..de97f8b27def --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/negative-nf.ok @@ -0,0 +1,2 @@ +../a.out: cannot set NF to a negative value + source line number 1 diff --git a/contrib/one-true-awk/bugs-fixed/pfile-overflow.ok b/contrib/one-true-awk/bugs-fixed/pfile-overflow.ok new file mode 100644 index 000000000000..a0de50f9007f --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/pfile-overflow.ok @@ -0,0 +1,4 @@ +../a.out: syntax error at source line 1 source file pfile-overflow.awk + context is + >>> <<< +../a.out: bailing out at source line 1 source file pfile-overflow.awk diff --git a/contrib/one-true-awk/bugs-fixed/rstart-rlength.awk b/contrib/one-true-awk/bugs-fixed/rstart-rlength.awk new file mode 100644 index 000000000000..f423f0168be3 --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/rstart-rlength.awk @@ -0,0 +1,10 @@ +BEGIN { + str="\342\200\257" + print length(str) + match(str,/^/) + print RSTART, RLENGTH + match(str,/.+/) + print RSTART, RLENGTH + match(str,/$/) + print RSTART, RLENGTH +} diff --git a/contrib/one-true-awk/bugs-fixed/rstart-rlength.ok b/contrib/one-true-awk/bugs-fixed/rstart-rlength.ok new file mode 100644 index 000000000000..961cb895b51b --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/rstart-rlength.ok @@ -0,0 +1,4 @@ +1 +1 0 +1 1 +2 0 diff --git a/contrib/one-true-awk/bugs-fixed/system-status.awk b/contrib/one-true-awk/bugs-fixed/system-status.awk new file mode 100644 index 000000000000..8daf563e6f4f --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/system-status.awk @@ -0,0 +1,19 @@ +# Unmodified nawk prints the 16 bit exit status divided by 256, but +# does so using floating point arithmetic, yielding strange results. +# +# The fix is to use the various macros defined for wait(2) and to +# use the signal number + 256 for death by signal, or signal number + 512 +# for death by signal with core dump. + +BEGIN { + status = system("exit 42") + print "normal status", status + + status = system("kill -HUP $$") + print "death by signal status", status + + status = system("kill -ABRT $$") + print "death by signal with core dump status", status + + system("rm -f core*") +} diff --git a/contrib/one-true-awk/bugs-fixed/system-status.bad b/contrib/one-true-awk/bugs-fixed/system-status.bad new file mode 100644 index 000000000000..a1317dba54a8 --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/system-status.bad @@ -0,0 +1,3 @@ +normal status 42 +death by signal status 0.00390625 +death by signal with core dump status 0.523438 diff --git a/contrib/one-true-awk/bugs-fixed/system-status.ok b/contrib/one-true-awk/bugs-fixed/system-status.ok new file mode 100644 index 000000000000..737828f5ed7a --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/system-status.ok @@ -0,0 +1,3 @@ +normal status 42 +death by signal status 257 +death by signal with core dump status 518 diff --git a/contrib/one-true-awk/bugs-fixed/system-status.ok2 b/contrib/one-true-awk/bugs-fixed/system-status.ok2 new file mode 100644 index 000000000000..f1f631e1cb33 --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/system-status.ok2 @@ -0,0 +1,3 @@ +normal status 42 +death by signal status 257 +death by signal with core dump status 262 diff --git a/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-1.awk b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-1.awk new file mode 100644 index 000000000000..67366ec75070 --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-1.awk @@ -0,0 +1,6 @@ +BEGIN { + FS="␟" + RS="␞" + OFS="," +} +{ print $1, $2, $3 } diff --git a/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-1.in b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-1.in new file mode 100644 index 000000000000..2e882af62a2c --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-1.in @@ -0,0 +1,2 @@ +id␟name␟age␞1␟Bob "Billy" Smith␟42␞2␟Jane + Brown␟37 diff --git a/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-1.ok b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-1.ok new file mode 100644 index 000000000000..f337302be903 --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-1.ok @@ -0,0 +1,5 @@ +id,name,age +1,Bob "Billy" Smith,42 +2,Jane + Brown,37 + diff --git a/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-2.awk b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-2.awk new file mode 100644 index 000000000000..34d77bf2c95f --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-2.awk @@ -0,0 +1,7 @@ +BEGIN { + FS = "א" + RS = "בב" + OFS = "," +} + +{ print $1, $2, $3 } diff --git a/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-2.in b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-2.in new file mode 100644 index 000000000000..2de6e718fd3b --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-2.in @@ -0,0 +1,2 @@ +idאnameאageא1אBob "Billy" Smithא42א2בבJane + Brownא37 diff --git a/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-2.ok b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-2.ok new file mode 100644 index 000000000000..2387001bc1b2 --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/unicode-fs-rs-2.ok @@ -0,0 +1,4 @@ +id,name,age +Jane + Brown,37 +, diff --git a/contrib/one-true-awk/bugs-fixed/unicode-null-match.awk b/contrib/one-true-awk/bugs-fixed/unicode-null-match.awk new file mode 100644 index 000000000000..0c056126922b --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/unicode-null-match.awk @@ -0,0 +1,6 @@ +BEGIN { + # str = "\342\200\257" + str = "あ" + n = gsub(//, "X", str) + print n, str +} diff --git a/contrib/one-true-awk/bugs-fixed/unicode-null-match.bad b/contrib/one-true-awk/bugs-fixed/unicode-null-match.bad new file mode 100644 index 000000000000..7cd35ff2d932 --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/unicode-null-match.bad @@ -0,0 +1 @@ +4 XXXX diff --git a/contrib/one-true-awk/bugs-fixed/unicode-null-match.ok b/contrib/one-true-awk/bugs-fixed/unicode-null-match.ok new file mode 100644 index 000000000000..1ac142f8a895 --- /dev/null +++ b/contrib/one-true-awk/bugs-fixed/unicode-null-match.ok @@ -0,0 +1 @@ +2 XあX diff --git a/contrib/one-true-awk/lex.c b/contrib/one-true-awk/lex.c index d471ac2944a9..141cc81d2b59 100644 --- a/contrib/one-true-awk/lex.c +++ b/contrib/one-true-awk/lex.c @@ -1,610 +1,646 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ #include #include #include #include #include "awk.h" #include "awkgram.tab.h" extern YYSTYPE yylval; extern bool infunc; int lineno = 1; int bracecnt = 0; int brackcnt = 0; int parencnt = 0; typedef struct Keyword { const char *word; int sub; int type; } Keyword; const Keyword keywords[] = { /* keep sorted: binary searched */ { "BEGIN", XBEGIN, XBEGIN }, { "END", XEND, XEND }, { "NF", VARNF, VARNF }, { "and", FAND, BLTIN }, { "atan2", FATAN, BLTIN }, { "break", BREAK, BREAK }, { "close", CLOSE, CLOSE }, { "compl", FCOMPL, BLTIN }, { "continue", CONTINUE, CONTINUE }, { "cos", FCOS, BLTIN }, { "delete", DELETE, DELETE }, { "do", DO, DO }, { "else", ELSE, ELSE }, { "exit", EXIT, EXIT }, { "exp", FEXP, BLTIN }, { "fflush", FFLUSH, BLTIN }, { "for", FOR, FOR }, { "func", FUNC, FUNC }, { "function", FUNC, FUNC }, { "gensub", GENSUB, GENSUB }, { "getline", GETLINE, GETLINE }, { "gsub", GSUB, GSUB }, { "if", IF, IF }, { "in", IN, IN }, { "index", INDEX, INDEX }, { "int", FINT, BLTIN }, { "length", FLENGTH, BLTIN }, { "log", FLOG, BLTIN }, { "lshift", FLSHIFT, BLTIN }, { "match", MATCHFCN, MATCHFCN }, { "next", NEXT, NEXT }, { "nextfile", NEXTFILE, NEXTFILE }, { "or", FFOR, BLTIN }, { "print", PRINT, PRINT }, { "printf", PRINTF, PRINTF }, { "rand", FRAND, BLTIN }, { "return", RETURN, RETURN }, { "rshift", FRSHIFT, BLTIN }, { "sin", FSIN, BLTIN }, { "split", SPLIT, SPLIT }, { "sprintf", SPRINTF, SPRINTF }, { "sqrt", FSQRT, BLTIN }, { "srand", FSRAND, BLTIN }, { "strftime", FSTRFTIME, BLTIN }, { "sub", SUB, SUB }, { "substr", SUBSTR, SUBSTR }, { "system", FSYSTEM, BLTIN }, { "systime", FSYSTIME, BLTIN }, { "tolower", FTOLOWER, BLTIN }, { "toupper", FTOUPPER, BLTIN }, { "while", WHILE, WHILE }, { "xor", FXOR, BLTIN }, }; #define RET(x) { if(dbg)printf("lex %s\n", tokname(x)); return(x); } static int peek(void) { int c = input(); unput(c); return c; } static int gettok(char **pbuf, int *psz) /* get next input token */ { int c, retc; char *buf = *pbuf; int sz = *psz; char *bp = buf; c = input(); if (c == 0) return 0; buf[0] = c; buf[1] = 0; if (!isalnum(c) && c != '.' && c != '_') return c; *bp++ = c; if (isalpha(c) || c == '_') { /* it's a varname */ for ( ; (c = input()) != 0; ) { if (bp-buf >= sz) if (!adjbuf(&buf, &sz, bp-buf+2, 100, &bp, "gettok")) FATAL( "out of space for name %.10s...", buf ); if (isalnum(c) || c == '_') *bp++ = c; else { *bp = 0; unput(c); break; } } *bp = 0; retc = 'a'; /* alphanumeric */ } else { /* maybe it's a number, but could be . */ char *rem; /* read input until can't be a number */ for ( ; (c = input()) != 0; ) { if (bp-buf >= sz) if (!adjbuf(&buf, &sz, bp-buf+2, 100, &bp, "gettok")) FATAL( "out of space for number %.10s...", buf ); if (isdigit(c) || c == 'e' || c == 'E' || c == '.' || c == '+' || c == '-') *bp++ = c; else { unput(c); break; } } *bp = 0; strtod(buf, &rem); /* parse the number */ if (rem == buf) { /* it wasn't a valid number at all */ buf[1] = 0; /* return one character as token */ retc = (uschar)buf[0]; /* character is its own type */ unputstr(rem+1); /* put rest back for later */ } else { /* some prefix was a number */ unputstr(rem); /* put rest back for later */ rem[0] = 0; /* truncate buf after number part */ retc = '0'; /* type is number */ } } *pbuf = buf; *psz = sz; return retc; } int word(char *); int string(void); int regexpr(void); bool sc = false; /* true => return a } right now */ bool reg = false; /* true => return a REGEXPR now */ int yylex(void) { int c; static char *buf = NULL; static int bufsize = 5; /* BUG: setting this small causes core dump! */ if (buf == NULL && (buf = (char *) malloc(bufsize)) == NULL) FATAL( "out of space in yylex" ); if (sc) { sc = false; RET('}'); } if (reg) { reg = false; return regexpr(); } for (;;) { c = gettok(&buf, &bufsize); if (c == 0) return 0; if (isalpha(c) || c == '_') return word(buf); if (isdigit(c)) { char *cp = tostring(buf); double result; if (is_number(cp, & result)) yylval.cp = setsymtab(buf, cp, result, CON|NUM, symtab); else yylval.cp = setsymtab(buf, cp, 0.0, STR, symtab); free(cp); /* should this also have STR set? */ RET(NUMBER); } yylval.i = c; switch (c) { case '\n': /* {EOL} */ lineno++; RET(NL); case '\r': /* assume \n is coming */ case ' ': /* {WS}+ */ case '\t': break; case '#': /* #.* strip comments */ while ((c = input()) != '\n' && c != 0) ; unput(c); /* * Next line is a hack, itcompensates for * unput's treatment of \n. */ lineno++; break; case ';': RET(';'); case '\\': if (peek() == '\n') { input(); lineno++; } else if (peek() == '\r') { input(); input(); /* \n */ lineno++; } else { RET(c); } break; case '&': if (peek() == '&') { input(); RET(AND); } else RET('&'); case '|': if (peek() == '|') { input(); RET(BOR); } else RET('|'); case '!': if (peek() == '=') { input(); yylval.i = NE; RET(NE); } else if (peek() == '~') { input(); yylval.i = NOTMATCH; RET(MATCHOP); } else RET(NOT); case '~': yylval.i = MATCH; RET(MATCHOP); case '<': if (peek() == '=') { input(); yylval.i = LE; RET(LE); } else { yylval.i = LT; RET(LT); } case '=': if (peek() == '=') { input(); yylval.i = EQ; RET(EQ); } else { yylval.i = ASSIGN; RET(ASGNOP); } case '>': if (peek() == '=') { input(); yylval.i = GE; RET(GE); } else if (peek() == '>') { input(); yylval.i = APPEND; RET(APPEND); } else { yylval.i = GT; RET(GT); } case '+': if (peek() == '+') { input(); yylval.i = INCR; RET(INCR); } else if (peek() == '=') { input(); yylval.i = ADDEQ; RET(ASGNOP); } else RET('+'); case '-': if (peek() == '-') { input(); yylval.i = DECR; RET(DECR); } else if (peek() == '=') { input(); yylval.i = SUBEQ; RET(ASGNOP); } else RET('-'); case '*': if (peek() == '=') { /* *= */ input(); yylval.i = MULTEQ; RET(ASGNOP); } else if (peek() == '*') { /* ** or **= */ input(); /* eat 2nd * */ if (peek() == '=') { input(); yylval.i = POWEQ; RET(ASGNOP); } else { RET(POWER); } } else RET('*'); case '/': RET('/'); case '%': if (peek() == '=') { input(); yylval.i = MODEQ; RET(ASGNOP); } else RET('%'); case '^': if (peek() == '=') { input(); yylval.i = POWEQ; RET(ASGNOP); } else RET(POWER); case '$': /* BUG: awkward, if not wrong */ c = gettok(&buf, &bufsize); if (isalpha(c)) { if (strcmp(buf, "NF") == 0) { /* very special */ unputstr("(NF)"); RET(INDIRECT); } c = peek(); if (c == '(' || c == '[' || (infunc && isarg(buf) >= 0)) { unputstr(buf); RET(INDIRECT); } yylval.cp = setsymtab(buf, "", 0.0, STR|NUM, symtab); RET(IVAR); } else if (c == 0) { /* */ SYNTAX( "unexpected end of input after $" ); RET(';'); } else { unputstr(buf); RET(INDIRECT); } case '}': if (--bracecnt < 0) SYNTAX( "extra }" ); sc = true; RET(';'); case ']': if (--brackcnt < 0) SYNTAX( "extra ]" ); RET(']'); case ')': if (--parencnt < 0) SYNTAX( "extra )" ); RET(')'); case '{': bracecnt++; RET('{'); case '[': brackcnt++; RET('['); case '(': parencnt++; RET('('); case '"': return string(); /* BUG: should be like tran.c ? */ default: RET(c); } } } +extern int runetochar(char *str, int c); + int string(void) { int c, n; char *s, *bp; static char *buf = NULL; static int bufsz = 500; if (buf == NULL && (buf = (char *) malloc(bufsz)) == NULL) FATAL("out of space for strings"); for (bp = buf; (c = input()) != '"'; ) { if (!adjbuf(&buf, &bufsz, bp-buf+2, 500, &bp, "string")) FATAL("out of space for string %.10s...", buf); switch (c) { case '\n': case '\r': case 0: *bp = '\0'; SYNTAX( "non-terminated string %.10s...", buf ); if (c == 0) /* hopeless */ FATAL( "giving up" ); lineno++; break; case '\\': c = input(); switch (c) { case '\n': break; case '"': *bp++ = '"'; break; case 'n': *bp++ = '\n'; break; case 't': *bp++ = '\t'; break; case 'f': *bp++ = '\f'; break; case 'r': *bp++ = '\r'; break; case 'b': *bp++ = '\b'; break; case 'v': *bp++ = '\v'; break; case 'a': *bp++ = '\a'; break; case '\\': *bp++ = '\\'; break; case '0': case '1': case '2': /* octal: \d \dd \ddd */ case '3': case '4': case '5': case '6': case '7': n = c - '0'; if ((c = peek()) >= '0' && c < '8') { n = 8 * n + input() - '0'; if ((c = peek()) >= '0' && c < '8') n = 8 * n + input() - '0'; } *bp++ = n; break; - case 'x': /* hex \x0-9a-fA-F + */ - { char xbuf[100], *px; - for (px = xbuf; (c = input()) != 0 && px-xbuf < 100-2; ) { - if (isdigit(c) - || (c >= 'a' && c <= 'f') - || (c >= 'A' && c <= 'F')) - *px++ = c; - else + case 'x': /* hex \x0-9a-fA-F (exactly two) */ + { + int i; + + if (!isxdigit(peek())) { + unput(c); + break; + } + n = 0; + for (i = 0; i < 2; i++) { + c = input(); + if (c == 0) break; + if (isxdigit(c)) { + c = tolower(c); + n *= 16; + if (isdigit(c)) + n += (c - '0'); + else + n += 10 + (c - 'a'); + } else { + unput(c); + break; + } + } + if (i) + *bp++ = n; + break; + } + + case 'u': /* utf \u0-9a-fA-F (1..8) */ + { + int i; + + n = 0; + for (i = 0; i < 8; i++) { + c = input(); + if (!isxdigit(c) || c == 0) + break; + c = tolower(c); + n *= 16; + if (isdigit(c)) + n += (c - '0'); + else + n += 10 + (c - 'a'); } - *px = 0; unput(c); - sscanf(xbuf, "%x", (unsigned int *) &n); - *bp++ = n; + bp += runetochar(bp, n); break; } default: *bp++ = c; break; } break; default: *bp++ = c; break; } } *bp = 0; s = tostring(buf); *bp++ = ' '; *bp++ = '\0'; yylval.cp = setsymtab(buf, s, 0.0, CON|STR|DONTFREE, symtab); free(s); RET(STRING); } static int binsearch(char *w, const Keyword *kp, int n) { int cond, low, mid, high; low = 0; high = n - 1; while (low <= high) { mid = (low + high) / 2; if ((cond = strcmp(w, kp[mid].word)) < 0) high = mid - 1; else if (cond > 0) low = mid + 1; else return mid; } return -1; } int word(char *w) { const Keyword *kp; int c, n; n = binsearch(w, keywords, sizeof(keywords)/sizeof(keywords[0])); if (n != -1) { /* found in table */ kp = keywords + n; yylval.i = kp->sub; switch (kp->type) { /* special handling */ case BLTIN: if (kp->sub == FSYSTEM && safe) SYNTAX( "system is unsafe" ); RET(kp->type); case FUNC: if (infunc) SYNTAX( "illegal nested function" ); RET(kp->type); case RETURN: if (!infunc) SYNTAX( "return not in function" ); RET(kp->type); case VARNF: yylval.cp = setsymtab("NF", "", 0.0, NUM, symtab); RET(VARNF); default: RET(kp->type); } } c = peek(); /* look for '(' */ if (c != '(' && infunc && (n=isarg(w)) >= 0) { yylval.i = n; RET(ARG); } else { yylval.cp = setsymtab(w, "", 0.0, STR|NUM|DONTFREE, symtab); if (c == '(') { RET(CALL); } else { RET(VAR); } } } void startreg(void) /* next call to yylex will return a regular expression */ { reg = true; } int regexpr(void) { int c; static char *buf = NULL; static int bufsz = 500; char *bp; if (buf == NULL && (buf = (char *) malloc(bufsz)) == NULL) - FATAL("out of space for rex expr"); + FATAL("out of space for reg expr"); bp = buf; for ( ; (c = input()) != '/' && c != 0; ) { if (!adjbuf(&buf, &bufsz, bp-buf+3, 500, &bp, "regexpr")) FATAL("out of space for reg expr %.10s...", buf); if (c == '\n') { *bp = '\0'; SYNTAX( "newline in regular expression %.10s...", buf ); unput('\n'); break; } else if (c == '\\') { *bp++ = '\\'; *bp++ = input(); } else { *bp++ = c; } } *bp = 0; if (c == 0) SYNTAX("non-terminated regular expression %.10s...", buf); yylval.s = tostring(buf); unput('/'); RET(REGEXPR); } /* low-level lexical stuff, sort of inherited from lex */ char ebuf[300]; char *ep = ebuf; char yysbuf[100]; /* pushback buffer */ char *yysptr = yysbuf; FILE *yyin = NULL; int input(void) /* get next lexical input character */ { int c; extern char *lexprog; if (yysptr > yysbuf) c = (uschar)*--yysptr; else if (lexprog != NULL) { /* awk '...' */ if ((c = (uschar)*lexprog) != 0) lexprog++; } else /* awk -f ... */ c = pgetc(); if (c == EOF) c = 0; if (ep >= ebuf + sizeof ebuf) ep = ebuf; *ep = c; if (c != 0) { ep++; } return (c); } void unput(int c) /* put lexical character back on input */ { if (c == '\n') lineno--; if (yysptr >= yysbuf + sizeof(yysbuf)) FATAL("pushed back too much: %.20s...", yysbuf); *yysptr++ = c; if (--ep < ebuf) ep = ebuf + sizeof(ebuf) - 1; } void unputstr(const char *s) /* put a string back on input */ { int i; for (i = strlen(s)-1; i >= 0; i--) unput(s[i]); } diff --git a/contrib/one-true-awk/lib.c b/contrib/one-true-awk/lib.c index ee77207564b7..0dac1f929cf2 100644 --- a/contrib/one-true-awk/lib.c +++ b/contrib/one-true-awk/lib.c @@ -1,849 +1,931 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ #define DEBUG #include #include #include #include #include #include #include #include #include #include "awk.h" +extern int u8_nextlen(const char *s); + char EMPTY[] = { '\0' }; FILE *infile = NULL; bool innew; /* true = infile has not been read by readrec */ char *file = EMPTY; char *record; int recsize = RECSIZE; char *fields; int fieldssize = RECSIZE; Cell **fldtab; /* pointers to Cells */ static size_t len_inputFS = 0; static char *inputFS = NULL; /* FS at time of input, for field splitting */ #define MAXFLD 2 int nfields = MAXFLD; /* last allocated slot for $i */ bool donefld; /* true = implies rec broken into fields */ bool donerec; /* true = record is valid (no flds have changed) */ int lastfld = 0; /* last used field */ int argno = 1; /* current input argument number */ extern Awkfloat *ARGC; static Cell dollar0 = { OCELL, CFLD, NULL, EMPTY, 0.0, REC|STR|DONTFREE, NULL, NULL }; static Cell dollar1 = { OCELL, CFLD, NULL, EMPTY, 0.0, FLD|STR|DONTFREE, NULL, NULL }; void recinit(unsigned int n) { if ( (record = (char *) malloc(n)) == NULL || (fields = (char *) malloc(n+1)) == NULL || (fldtab = (Cell **) calloc(nfields+2, sizeof(*fldtab))) == NULL || (fldtab[0] = (Cell *) malloc(sizeof(**fldtab))) == NULL) FATAL("out of space for $0 and fields"); *record = '\0'; *fldtab[0] = dollar0; fldtab[0]->sval = record; fldtab[0]->nval = tostring("0"); makefields(1, nfields); } void makefields(int n1, int n2) /* create $n1..$n2 inclusive */ { char temp[50]; int i; for (i = n1; i <= n2; i++) { fldtab[i] = (Cell *) malloc(sizeof(**fldtab)); if (fldtab[i] == NULL) FATAL("out of space in makefields %d", i); *fldtab[i] = dollar1; snprintf(temp, sizeof(temp), "%d", i); fldtab[i]->nval = tostring(temp); } } void initgetrec(void) { int i; char *p; for (i = 1; i < *ARGC; i++) { p = getargv(i); /* find 1st real filename */ if (p == NULL || *p == '\0') { /* deleted or zapped */ argno++; continue; } if (!isclvar(p)) { setsval(lookup("FILENAME", symtab), p); return; } setclvar(p); /* a commandline assignment before filename */ argno++; } infile = stdin; /* no filenames, so use stdin */ innew = true; } /* * POSIX specifies that fields are supposed to be evaluated as if they were * split using the value of FS at the time that the record's value ($0) was * read. * * Since field-splitting is done lazily, we save the current value of FS * whenever a new record is read in (implicitly or via getline), or when * a new value is assigned to $0. */ void savefs(void) { size_t len; if ((len = strlen(getsval(fsloc))) < len_inputFS) { strcpy(inputFS, *FS); /* for subsequent field splitting */ return; } len_inputFS = len + 1; inputFS = (char *) realloc(inputFS, len_inputFS); if (inputFS == NULL) FATAL("field separator %.10s... is too long", *FS); memcpy(inputFS, *FS, len_inputFS); } static bool firsttime = true; int getrec(char **pbuf, int *pbufsize, bool isrecord) /* get next input record */ { /* note: cares whether buf == record */ int c; char *buf = *pbuf; uschar saveb0; int bufsize = *pbufsize, savebufsize = bufsize; if (firsttime) { firsttime = false; initgetrec(); } DPRINTF("RS=<%s>, FS=<%s>, ARGC=%g, FILENAME=%s\n", *RS, *FS, *ARGC, *FILENAME); - if (isrecord) { - donefld = false; - donerec = true; - savefs(); - } saveb0 = buf[0]; buf[0] = 0; while (argno < *ARGC || infile == stdin) { DPRINTF("argno=%d, file=|%s|\n", argno, file); if (infile == NULL) { /* have to open a new file */ file = getargv(argno); if (file == NULL || *file == '\0') { /* deleted or zapped */ argno++; continue; } if (isclvar(file)) { /* a var=value arg */ setclvar(file); argno++; continue; } *FILENAME = file; DPRINTF("opening file %s\n", file); if (*file == '-' && *(file+1) == '\0') infile = stdin; else if ((infile = fopen(file, "r")) == NULL) FATAL("can't open file %s", file); innew = true; setfval(fnrloc, 0.0); } c = readrec(&buf, &bufsize, infile, innew); if (innew) innew = false; if (c != 0 || buf[0] != '\0') { /* normal record */ if (isrecord) { double result; if (freeable(fldtab[0])) xfree(fldtab[0]->sval); fldtab[0]->sval = buf; /* buf == record */ fldtab[0]->tval = REC | STR | DONTFREE; if (is_number(fldtab[0]->sval, & result)) { fldtab[0]->fval = result; fldtab[0]->tval |= NUM; } + donefld = false; + donerec = true; + savefs(); } setfval(nrloc, nrloc->fval+1); setfval(fnrloc, fnrloc->fval+1); *pbuf = buf; *pbufsize = bufsize; return 1; } /* EOF arrived on this file; set up next */ if (infile != stdin) fclose(infile); infile = NULL; argno++; } buf[0] = saveb0; *pbuf = buf; *pbufsize = savebufsize; return 0; /* true end of file */ } void nextfile(void) { if (infile != NULL && infile != stdin) fclose(infile); infile = NULL; argno++; } +extern int readcsvrec(char **pbuf, int *pbufsize, FILE *inf, bool newflag); + int readrec(char **pbuf, int *pbufsize, FILE *inf, bool newflag) /* read one record into buf */ { - int sep, c, isrec; - char *rr, *buf = *pbuf; + int sep, c, isrec; // POTENTIAL BUG? isrec is a macro in awk.h + char *rr = *pbuf, *buf = *pbuf; int bufsize = *pbufsize; char *rs = getsval(rsloc); - if (*rs && rs[1]) { + if (CSV) { + c = readcsvrec(pbuf, pbufsize, inf, newflag); + isrec = (c == EOF && rr == buf) ? false : true; + } else if (*rs && rs[1]) { bool found; + memset(buf, 0, bufsize); fa *pfa = makedfa(rs, 1); if (newflag) found = fnematch(pfa, inf, &buf, &bufsize, recsize); else { int tempstat = pfa->initstat; pfa->initstat = 2; found = fnematch(pfa, inf, &buf, &bufsize, recsize); pfa->initstat = tempstat; } if (found) setptr(patbeg, '\0'); isrec = (found == 0 && *buf == '\0') ? false : true; + } else { if ((sep = *rs) == 0) { sep = '\n'; while ((c=getc(inf)) == '\n' && c != EOF) /* skip leading \n's */ ; if (c != EOF) ungetc(c, inf); } for (rr = buf; ; ) { for (; (c=getc(inf)) != sep && c != EOF; ) { if (rr-buf+1 > bufsize) if (!adjbuf(&buf, &bufsize, 1+rr-buf, recsize, &rr, "readrec 1")) FATAL("input record `%.30s...' too long", buf); *rr++ = c; } if (*rs == sep || c == EOF) break; if ((c = getc(inf)) == '\n' || c == EOF) /* 2 in a row */ break; if (!adjbuf(&buf, &bufsize, 2+rr-buf, recsize, &rr, "readrec 2")) FATAL("input record `%.30s...' too long", buf); *rr++ = '\n'; *rr++ = c; } if (!adjbuf(&buf, &bufsize, 1+rr-buf, recsize, &rr, "readrec 3")) FATAL("input record `%.30s...' too long", buf); *rr = 0; isrec = (c == EOF && rr == buf) ? false : true; } *pbuf = buf; *pbufsize = bufsize; DPRINTF("readrec saw <%s>, returns %d\n", buf, isrec); return isrec; } + +/******************* + * loose ends here: + * \r\n should become \n + * what about bare \r? Excel uses that for embedded newlines + * can't have "" in unquoted fields, according to RFC 4180 +*/ + + +int readcsvrec(char **pbuf, int *pbufsize, FILE *inf, bool newflag) /* csv can have \n's */ +{ /* so read a complete record that might be multiple lines */ + int sep, c; + char *rr = *pbuf, *buf = *pbuf; + int bufsize = *pbufsize; + bool in_quote = false; + + sep = '\n'; /* the only separator; have to skip over \n embedded in "..." */ + rr = buf; + while ((c = getc(inf)) != EOF) { + if (c == sep) { + if (! in_quote) + break; + if (rr > buf && rr[-1] == '\r') // remove \r if was \r\n + rr--; + } + + if (rr-buf+1 > bufsize) + if (!adjbuf(&buf, &bufsize, 1+rr-buf, + recsize, &rr, "readcsvrec 1")) + FATAL("input record `%.30s...' too long", buf); + *rr++ = c; + if (c == '"') + in_quote = ! in_quote; + } + if (c == '\n' && rr > buf && rr[-1] == '\r') // remove \r if was \r\n + rr--; + + if (!adjbuf(&buf, &bufsize, 1+rr-buf, recsize, &rr, "readcsvrec 4")) + FATAL("input record `%.30s...' too long", buf); + *rr = 0; + *pbuf = buf; + *pbufsize = bufsize; + DPRINTF("readcsvrec saw <%s>, returns %d\n", buf, c); + return c; +} + char *getargv(int n) /* get ARGV[n] */ { Cell *x; char *s, temp[50]; extern Array *ARGVtab; snprintf(temp, sizeof(temp), "%d", n); if (lookup(temp, ARGVtab) == NULL) return NULL; x = setsymtab(temp, "", 0.0, STR, ARGVtab); s = getsval(x); DPRINTF("getargv(%d) returns |%s|\n", n, s); return s; } void setclvar(char *s) /* set var=value from s */ { - char *p; + char *e, *p; Cell *q; double result; +/* commit f3d9187d4e0f02294fb1b0e31152070506314e67 broke T.argv test */ +/* I don't understand why it was changed. */ + for (p=s; *p != '='; p++) ; + e = p; *p++ = 0; p = qstring(p, '\0'); q = setsymtab(s, p, 0.0, STR, symtab); setsval(q, p); if (is_number(q->sval, & result)) { q->fval = result; q->tval |= NUM; } DPRINTF("command line set %s to |%s|\n", s, p); + free(p); + *e = '='; } void fldbld(void) /* create fields from current record */ { /* this relies on having fields[] the same length as $0 */ /* the fields are all stored in this one array with \0's */ /* possibly with a final trailing \0 not associated with any field */ char *r, *fr, sep; Cell *p; int i, j, n; if (donefld) return; if (!isstr(fldtab[0])) getsval(fldtab[0]); r = fldtab[0]->sval; n = strlen(r); if (n > fieldssize) { xfree(fields); if ((fields = (char *) malloc(n+2)) == NULL) /* possibly 2 final \0s */ FATAL("out of space for fields in fldbld %d", n); fieldssize = n; } fr = fields; i = 0; /* number of fields accumulated here */ if (inputFS == NULL) /* make sure we have a copy of FS */ savefs(); - if (strlen(inputFS) > 1) { /* it's a regular expression */ + if (!CSV && strlen(inputFS) > 1) { /* it's a regular expression */ i = refldbld(r, inputFS); - } else if ((sep = *inputFS) == ' ') { /* default whitespace */ + } else if (!CSV && (sep = *inputFS) == ' ') { /* default whitespace */ for (i = 0; ; ) { while (*r == ' ' || *r == '\t' || *r == '\n') r++; if (*r == 0) break; i++; if (i > nfields) growfldtab(i); if (freeable(fldtab[i])) xfree(fldtab[i]->sval); fldtab[i]->sval = fr; fldtab[i]->tval = FLD | STR | DONTFREE; do *fr++ = *r++; while (*r != ' ' && *r != '\t' && *r != '\n' && *r != '\0'); *fr++ = 0; } *fr = 0; - } else if ((sep = *inputFS) == 0) { /* new: FS="" => 1 char/field */ - for (i = 0; *r != '\0'; r += n) { - char buf[MB_LEN_MAX + 1]; - + } else if (CSV) { /* CSV processing. no error handling */ + if (*r != 0) { + for (;;) { + i++; + if (i > nfields) + growfldtab(i); + if (freeable(fldtab[i])) + xfree(fldtab[i]->sval); + fldtab[i]->sval = fr; + fldtab[i]->tval = FLD | STR | DONTFREE; + if (*r == '"' ) { /* start of "..." */ + for (r++ ; *r != '\0'; ) { + if (*r == '"' && r[1] != '\0' && r[1] == '"') { + r += 2; /* doubled quote */ + *fr++ = '"'; + } else if (*r == '"' && (r[1] == '\0' || r[1] == ',')) { + r++; /* skip over closing quote */ + break; + } else { + *fr++ = *r++; + } + } + *fr++ = 0; + } else { /* unquoted field */ + while (*r != ',' && *r != '\0') + *fr++ = *r++; + *fr++ = 0; + } + if (*r++ == 0) + break; + + } + } + *fr = 0; + } else if ((sep = *inputFS) == 0) { /* new: FS="" => 1 char/field */ + for (i = 0; *r != '\0'; ) { + char buf[10]; i++; if (i > nfields) growfldtab(i); if (freeable(fldtab[i])) xfree(fldtab[i]->sval); - n = mblen(r, MB_LEN_MAX); - if (n < 0) - n = 1; - memcpy(buf, r, n); - buf[n] = '\0'; + n = u8_nextlen(r); + for (j = 0; j < n; j++) + buf[j] = *r++; + buf[j] = '\0'; fldtab[i]->sval = tostring(buf); fldtab[i]->tval = FLD | STR; } *fr = 0; } else if (*r != 0) { /* if 0, it's a null field */ - /* subtlecase : if length(FS) == 1 && length(RS > 0) + /* subtle case: if length(FS) == 1 && length(RS > 0) * \n is NOT a field separator (cf awk book 61,84). * this variable is tested in the inner while loop. */ int rtest = '\n'; /* normal case */ if (strlen(*RS) > 0) rtest = '\0'; for (;;) { i++; if (i > nfields) growfldtab(i); if (freeable(fldtab[i])) xfree(fldtab[i]->sval); fldtab[i]->sval = fr; fldtab[i]->tval = FLD | STR | DONTFREE; while (*r != sep && *r != rtest && *r != '\0') /* \n is always a separator */ *fr++ = *r++; *fr++ = 0; if (*r++ == 0) break; } *fr = 0; } if (i > nfields) FATAL("record `%.30s...' has too many fields; can't happen", r); cleanfld(i+1, lastfld); /* clean out junk from previous record */ lastfld = i; donefld = true; for (j = 1; j <= lastfld; j++) { double result; p = fldtab[j]; if(is_number(p->sval, & result)) { p->fval = result; p->tval |= NUM; } } setfval(nfloc, (Awkfloat) lastfld); donerec = true; /* restore */ if (dbg) { for (j = 0; j <= lastfld; j++) { p = fldtab[j]; printf("field %d (%s): |%s|\n", j, p->nval, p->sval); } } } void cleanfld(int n1, int n2) /* clean out fields n1 .. n2 inclusive */ { /* nvals remain intact */ Cell *p; int i; for (i = n1; i <= n2; i++) { p = fldtab[i]; if (freeable(p)) xfree(p->sval); p->sval = EMPTY, p->tval = FLD | STR | DONTFREE; } } void newfld(int n) /* add field n after end of existing lastfld */ { if (n > nfields) growfldtab(n); cleanfld(lastfld+1, n); lastfld = n; setfval(nfloc, (Awkfloat) n); } void setlastfld(int n) /* set lastfld cleaning fldtab cells if necessary */ { if (n < 0) FATAL("cannot set NF to a negative value"); if (n > nfields) growfldtab(n); if (lastfld < n) cleanfld(lastfld+1, n); else cleanfld(n+1, lastfld); lastfld = n; } Cell *fieldadr(int n) /* get nth field */ { if (n < 0) FATAL("trying to access out of range field %d", n); if (n > nfields) /* fields after NF are empty */ growfldtab(n); /* but does not increase NF */ return(fldtab[n]); } void growfldtab(int n) /* make new fields up to at least $n */ { int nf = 2 * nfields; size_t s; if (n > nf) nf = n; s = (nf+1) * (sizeof (struct Cell *)); /* freebsd: how much do we need? */ if (s / sizeof(struct Cell *) - 1 == (size_t)nf) /* didn't overflow */ fldtab = (Cell **) realloc(fldtab, s); else /* overflow sizeof int */ xfree(fldtab); /* make it null */ if (fldtab == NULL) FATAL("out of space creating %d fields", nf); makefields(nfields+1, nf); nfields = nf; } int refldbld(const char *rec, const char *fs) /* build fields from reg expr in FS */ { /* this relies on having fields[] the same length as $0 */ /* the fields are all stored in this one array with \0's */ char *fr; int i, tempstat, n; fa *pfa; n = strlen(rec); if (n > fieldssize) { xfree(fields); if ((fields = (char *) malloc(n+1)) == NULL) FATAL("out of space for fields in refldbld %d", n); fieldssize = n; } fr = fields; *fr = '\0'; if (*rec == '\0') return 0; pfa = makedfa(fs, 1); DPRINTF("into refldbld, rec = <%s>, pat = <%s>\n", rec, fs); tempstat = pfa->initstat; for (i = 1; ; i++) { if (i > nfields) growfldtab(i); if (freeable(fldtab[i])) xfree(fldtab[i]->sval); fldtab[i]->tval = FLD | STR | DONTFREE; fldtab[i]->sval = fr; DPRINTF("refldbld: i=%d\n", i); if (nematch(pfa, rec)) { pfa->initstat = 2; /* horrible coupling to b.c */ DPRINTF("match %s (%d chars)\n", patbeg, patlen); strncpy(fr, rec, patbeg-rec); fr += patbeg - rec + 1; *(fr-1) = '\0'; rec = patbeg + patlen; } else { DPRINTF("no match %s\n", rec); strcpy(fr, rec); pfa->initstat = tempstat; break; } } return i; } void recbld(void) /* create $0 from $1..$NF if necessary */ { int i; char *r, *p; char *sep = getsval(ofsloc); if (donerec) return; r = record; for (i = 1; i <= *NF; i++) { p = getsval(fldtab[i]); if (!adjbuf(&record, &recsize, 1+strlen(p)+r-record, recsize, &r, "recbld 1")) FATAL("created $0 `%.30s...' too long", record); while ((*r = *p++) != 0) r++; if (i < *NF) { if (!adjbuf(&record, &recsize, 2+strlen(sep)+r-record, recsize, &r, "recbld 2")) FATAL("created $0 `%.30s...' too long", record); for (p = sep; (*r = *p++) != 0; ) r++; } } if (!adjbuf(&record, &recsize, 2+r-record, recsize, &r, "recbld 3")) FATAL("built giant record `%.30s...'", record); *r = '\0'; DPRINTF("in recbld inputFS=%s, fldtab[0]=%p\n", inputFS, (void*)fldtab[0]); if (freeable(fldtab[0])) xfree(fldtab[0]->sval); fldtab[0]->tval = REC | STR | DONTFREE; fldtab[0]->sval = record; DPRINTF("in recbld inputFS=%s, fldtab[0]=%p\n", inputFS, (void*)fldtab[0]); DPRINTF("recbld = |%s|\n", record); donerec = true; } int errorflag = 0; void yyerror(const char *s) { SYNTAX("%s", s); } void SYNTAX(const char *fmt, ...) { extern char *cmdname, *curfname; static int been_here = 0; va_list varg; if (been_here++ > 2) return; fprintf(stderr, "%s: ", cmdname); va_start(varg, fmt); vfprintf(stderr, fmt, varg); va_end(varg); fprintf(stderr, " at source line %d", lineno); if (curfname != NULL) fprintf(stderr, " in function %s", curfname); if (compile_time == COMPILING && cursource() != NULL) fprintf(stderr, " source file %s", cursource()); fprintf(stderr, "\n"); errorflag = 2; eprint(); } extern int bracecnt, brackcnt, parencnt; void bracecheck(void) { int c; static int beenhere = 0; if (beenhere++) return; while ((c = input()) != EOF && c != '\0') bclass(c); bcheck2(bracecnt, '{', '}'); bcheck2(brackcnt, '[', ']'); bcheck2(parencnt, '(', ')'); } void bcheck2(int n, int c1, int c2) { if (n == 1) fprintf(stderr, "\tmissing %c\n", c2); else if (n > 1) fprintf(stderr, "\t%d missing %c's\n", n, c2); else if (n == -1) fprintf(stderr, "\textra %c\n", c2); else if (n < -1) fprintf(stderr, "\t%d extra %c's\n", -n, c2); } void FATAL(const char *fmt, ...) { extern char *cmdname; va_list varg; fflush(stdout); fprintf(stderr, "%s: ", cmdname); va_start(varg, fmt); vfprintf(stderr, fmt, varg); va_end(varg); error(); if (dbg > 1) /* core dump if serious debugging on */ abort(); exit(2); } void WARNING(const char *fmt, ...) { extern char *cmdname; va_list varg; fflush(stdout); fprintf(stderr, "%s: ", cmdname); va_start(varg, fmt); vfprintf(stderr, fmt, varg); va_end(varg); error(); } void error() { extern Node *curnode; fprintf(stderr, "\n"); if (compile_time != ERROR_PRINTING) { if (NR && *NR > 0) { fprintf(stderr, " input record number %d", (int) (*FNR)); if (strcmp(*FILENAME, "-") != 0) fprintf(stderr, ", file %s", *FILENAME); fprintf(stderr, "\n"); } if (curnode) fprintf(stderr, " source line number %d", curnode->lineno); else if (lineno) fprintf(stderr, " source line number %d", lineno); if (compile_time == COMPILING && cursource() != NULL) fprintf(stderr, " source file %s", cursource()); fprintf(stderr, "\n"); eprint(); } } void eprint(void) /* try to print context around error */ { char *p, *q; int c; static int been_here = 0; extern char ebuf[], *ep; if (compile_time != COMPILING || been_here++ > 0 || ebuf == ep) return; if (ebuf == ep) return; p = ep - 1; if (p > ebuf && *p == '\n') p--; for ( ; p > ebuf && *p != '\n' && *p != '\0'; p--) ; while (*p == '\n') p++; fprintf(stderr, " context is\n\t"); for (q=ep-1; q>=p && *q!=' ' && *q!='\t' && *q!='\n'; q--) ; for ( ; p < q; p++) if (*p) putc(*p, stderr); fprintf(stderr, " >>> "); for ( ; p < ep; p++) if (*p) putc(*p, stderr); fprintf(stderr, " <<< "); if (*ep) while ((c = input()) != '\n' && c != '\0' && c != EOF) { putc(c, stderr); bclass(c); } putc('\n', stderr); ep = ebuf; } void bclass(int c) { switch (c) { case '{': bracecnt++; break; case '}': bracecnt--; break; case '[': brackcnt++; break; case ']': brackcnt--; break; case '(': parencnt++; break; case ')': parencnt--; break; } } double errcheck(double x, const char *s) { if (errno == EDOM) { errno = 0; WARNING("%s argument out of domain", s); x = 1; } else if (errno == ERANGE) { errno = 0; WARNING("%s result out of range", s); x = 1; } return x; } int isclvar(const char *s) /* is s of form var=something ? */ { const char *os = s; - if (!isalpha((uschar) *s) && *s != '_') + if (!isalpha((int) *s) && *s != '_') return 0; for ( ; *s; s++) - if (!(isalnum((uschar) *s) || *s == '_')) + if (!(isalnum((int) *s) || *s == '_')) break; return *s == '=' && s > os; } /* strtod is supposed to be a proper test of what's a valid number */ /* appears to be broken in gcc on linux: thinks 0x123 is a valid FP number */ /* wrong: violates 4.10.1.4 of ansi C standard */ /* well, not quite. As of C99, hex floating point is allowed. so this is * a bit of a mess. We work around the mess by checking for a hexadecimal * value and disallowing it. Similarly, we now follow gawk and allow only * +nan, -nan, +inf, and -inf for NaN and infinity values. */ /* * This routine now has a more complicated interface, the main point * being to avoid the double conversion of a string to double, and * also to convey out, if requested, the information that the numeric * value was a leading string or is all of the string. The latter bit * is used in getfval(). */ bool is_valid_number(const char *s, bool trailing_stuff_ok, bool *no_trailing, double *result) { double r; char *ep; bool retval = false; bool is_nan = false; bool is_inf = false; if (no_trailing) *no_trailing = false; - while (isspace(*s)) + while (isspace((int) *s)) s++; -/* - * This test, while allowed by newer POSIX standards, represents a regression - * where hex strings were treated as numbers in nawk the whole time it has been - * in FreeBSD (since 2001). The POSIX 2001 through 2004 standards mandated this - * behavior and the current standard allows it. Deviate from upstream by restoring - * the prior FreeBSD behavior. - */ -#if 0 - // no hex floating point, sorry + /* no hex floating point, sorry */ if (s[0] == '0' && tolower(s[1]) == 'x') return false; -#endif - // allow +nan, -nan, +inf, -inf, any other letter, no + /* allow +nan, -nan, +inf, -inf, any other letter, no */ if (s[0] == '+' || s[0] == '-') { is_nan = (strncasecmp(s+1, "nan", 3) == 0); is_inf = (strncasecmp(s+1, "inf", 3) == 0); if ((is_nan || is_inf) - && (isspace(s[4]) || s[4] == '\0')) + && (isspace((int) s[4]) || s[4] == '\0')) goto convert; else if (! isdigit(s[1]) && s[1] != '.') return false; } else if (! isdigit(s[0]) && s[0] != '.') return false; convert: errno = 0; r = strtod(s, &ep); if (ep == s || errno == ERANGE) return false; if (isnan(r) && s[0] == '-' && signbit(r) == 0) r = -r; if (result != NULL) *result = r; /* * check for trailing stuff */ - while (isspace(*ep)) + while (isspace((int) *ep)) ep++; if (no_trailing != NULL) *no_trailing = (*ep == '\0'); - // return true if found the end, or trailing stuff is allowed + /* return true if found the end, or trailing stuff is allowed */ retval = *ep == '\0' || trailing_stuff_ok; return retval; } diff --git a/contrib/one-true-awk/main.c b/contrib/one-true-awk/main.c index daa030000a51..73af89ec1058 100644 --- a/contrib/one-true-awk/main.c +++ b/contrib/one-true-awk/main.c @@ -1,259 +1,272 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ -const char *version = "version 20210724"; +const char *version = "version 20240122"; #define DEBUG #include #include #include #include #include #include #include "awk.h" extern char **environ; extern int nfields; int dbg = 0; Awkfloat srand_seed = 1; char *cmdname; /* gets argv[0] for error messages */ extern FILE *yyin; /* lex input file */ char *lexprog; /* points to program argument if it exists */ extern int errorflag; /* non-zero if any syntax errors; set by yyerror */ enum compile_states compile_time = ERROR_PRINTING; static char **pfile; /* program filenames from -f's */ static size_t maxpfile; /* max program filename */ static size_t npfile; /* number of filenames */ static size_t curpfile; /* current filename */ +bool CSV = false; /* true for csv input */ + bool safe = false; /* true => "safe" mode */ +size_t awk_mb_cur_max = 1; + static noreturn void fpecatch(int n #ifdef SA_SIGINFO , siginfo_t *si, void *uc #endif ) { #ifdef SA_SIGINFO static const char *emsg[] = { [0] = "Unknown error", [FPE_INTDIV] = "Integer divide by zero", [FPE_INTOVF] = "Integer overflow", [FPE_FLTDIV] = "Floating point divide by zero", [FPE_FLTOVF] = "Floating point overflow", [FPE_FLTUND] = "Floating point underflow", [FPE_FLTRES] = "Floating point inexact result", [FPE_FLTINV] = "Invalid Floating point operation", [FPE_FLTSUB] = "Subscript out of range", }; #endif FATAL("floating point exception" #ifdef SA_SIGINFO ": %s", (size_t)si->si_code < sizeof(emsg) / sizeof(emsg[0]) && emsg[si->si_code] ? emsg[si->si_code] : emsg[0] #endif ); } /* Can this work with recursive calls? I don't think so. void segvcatch(int n) { FATAL("segfault. Do you have an unbounded recursive call?", n); } */ static const char * setfs(char *p) { /* wart: t=>\t */ - if (p[0] == 't' && p[1] == '\0') { - WARNING("-Ft to imply tab separator is deprecated behavior."); + if (p[0] == 't' && p[1] == '\0') return "\t"; - } return p; } static char * getarg(int *argc, char ***argv, const char *msg) { if ((*argv)[1][2] != '\0') { /* arg is -fsomething */ return &(*argv)[1][2]; } else { /* arg is -f something */ (*argc)--; (*argv)++; if (*argc <= 1) FATAL("%s", msg); return (*argv)[1]; } } int main(int argc, char *argv[]) { const char *fs = NULL; char *fn, *vn; setlocale(LC_CTYPE, ""); setlocale(LC_NUMERIC, "C"); /* for parsing cmdline & prog */ + awk_mb_cur_max = MB_CUR_MAX; cmdname = argv[0]; if (argc == 1) { fprintf(stderr, - "usage: %s [-F fs] [-v var=value] [-f progfile | 'prog'] [file ...]\n", + "usage: %s [-F fs | --csv] [-v var=value] [-f progfile | 'prog'] [file ...]\n", cmdname); exit(1); } #ifdef SA_SIGINFO { struct sigaction sa; sa.sa_sigaction = fpecatch; sa.sa_flags = SA_SIGINFO; sigemptyset(&sa.sa_mask); (void)sigaction(SIGFPE, &sa, NULL); } #else (void)signal(SIGFPE, fpecatch); #endif /*signal(SIGSEGV, segvcatch); experiment */ /* Set and keep track of the random seed */ srand_seed = 1; srandom((unsigned long) srand_seed); yyin = NULL; symtab = makesymtab(NSYMTAB/NSYMTAB); while (argc > 1 && argv[1][0] == '-' && argv[1][1] != '\0') { if (strcmp(argv[1], "-version") == 0 || strcmp(argv[1], "--version") == 0) { printf("awk %s\n", version); return 0; } if (strcmp(argv[1], "--") == 0) { /* explicit end of args */ argc--; argv++; break; } + if (strcmp(argv[1], "--csv") == 0) { /* turn on csv input processing */ + CSV = true; + argc--; + argv++; + continue; + } switch (argv[1][1]) { case 's': if (strcmp(argv[1], "-safe") == 0) safe = true; break; case 'f': /* next argument is program filename */ fn = getarg(&argc, &argv, "no program filename"); if (npfile >= maxpfile) { maxpfile += 20; pfile = (char **) realloc(pfile, maxpfile * sizeof(*pfile)); if (pfile == NULL) FATAL("error allocating space for -f options"); } pfile[npfile++] = fn; break; case 'F': /* set field separator */ fs = setfs(getarg(&argc, &argv, "no field separator")); break; case 'v': /* -v a=1 to be done NOW. one -v for each */ vn = getarg(&argc, &argv, "no variable name"); if (isclvar(vn)) setclvar(vn); else FATAL("invalid -v option argument: %s", vn); break; case 'd': dbg = atoi(&argv[1][2]); if (dbg == 0) dbg = 1; printf("awk %s\n", version); break; default: WARNING("unknown option %s ignored", argv[1]); break; } argc--; argv++; } + + if (CSV && (fs != NULL || lookup("FS", symtab) != NULL)) + WARNING("danger: don't set FS when --csv is in effect"); + /* argv[1] is now the first argument */ if (npfile == 0) { /* no -f; first argument is program */ if (argc <= 1) { if (dbg) exit(0); FATAL("no program given"); } DPRINTF("program = |%s|\n", argv[1]); lexprog = argv[1]; argc--; argv++; } recinit(recsize); syminit(); compile_time = COMPILING; argv[0] = cmdname; /* put prog name at front of arglist */ DPRINTF("argc=%d, argv[0]=%s\n", argc, argv[0]); arginit(argc, argv); if (!safe) envinit(environ); yyparse(); #if 0 // Doing this would comply with POSIX, but is not compatible with // other awks and with what most users expect. So comment it out. setlocale(LC_NUMERIC, ""); /* back to whatever it is locally */ #endif if (fs) *FS = qstring(fs, '\0'); DPRINTF("errorflag=%d\n", errorflag); if (errorflag == 0) { compile_time = RUNNING; run(winner); } else bracecheck(); return(errorflag); } int pgetc(void) /* get 1 character from awk program */ { int c; for (;;) { if (yyin == NULL) { if (curpfile >= npfile) return EOF; if (strcmp(pfile[curpfile], "-") == 0) yyin = stdin; else if ((yyin = fopen(pfile[curpfile], "r")) == NULL) FATAL("can't open file %s", pfile[curpfile]); lineno = 1; } if ((c = getc(yyin)) != EOF) return c; if (yyin != stdin) fclose(yyin); yyin = NULL; curpfile++; } } char *cursource(void) /* current source file name */ { if (npfile > 0) return pfile[curpfile < npfile ? curpfile : curpfile - 1]; else return NULL; } diff --git a/contrib/one-true-awk/makefile b/contrib/one-true-awk/makefile index 9ceaaad48af1..c62908709d1c 100644 --- a/contrib/one-true-awk/makefile +++ b/contrib/one-true-awk/makefile @@ -1,111 +1,111 @@ # /**************************************************************** # Copyright (C) Lucent Technologies 1997 # All Rights Reserved # # Permission to use, copy, modify, and distribute this software and # its documentation for any purpose and without fee is hereby # granted, provided that the above copyright notice appear in all # copies and that both that the copyright notice and this # permission notice and warranty disclaimer appear in supporting # documentation, and that the name Lucent Technologies or any of # its entities not be used in advertising or publicity pertaining # to distribution of the software without specific, written prior # permission. # # LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, # INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. # IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY # SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER # IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, # ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF # THIS SOFTWARE. # ****************************************************************/ CFLAGS = -fsanitize=address -O1 -g -fno-omit-frame-pointer -fno-optimize-sibling-calls CFLAGS = -g CFLAGS = CFLAGS = -O2 # compiler options -#CC = gcc -Wall -g -Wwrite-strings -#CC = gcc -O4 -Wall -pedantic -fno-strict-aliasing -#CC = gcc -fprofile-arcs -ftest-coverage # then gcov f1.c; cat f1.c.gcov -HOSTCC = gcc -g -Wall -pedantic -Wcast-qual +#CC = cc -Wall -g -Wwrite-strings +#CC = cc -O4 -Wall -pedantic -fno-strict-aliasing +#CC = cc -fprofile-arcs -ftest-coverage # then gcov f1.c; cat f1.c.gcov +HOSTCC = cc -g -Wall -pedantic -Wcast-qual CC = $(HOSTCC) # change this is cross-compiling. # By fiat, to make our lives easier, yacc is now defined to be bison. # If you want something else, you're on your own. YACC = bison -d OFILES = b.o main.o parse.o proctab.o tran.o lib.o run.o lex.o SOURCE = awk.h awkgram.tab.c awkgram.tab.h proto.h awkgram.y lex.c b.c main.c \ maketab.c parse.c lib.c run.c tran.c proctab.c LISTING = awk.h proto.h awkgram.y lex.c b.c main.c maketab.c parse.c \ lib.c run.c tran.c SHIP = README LICENSE FIXES $(SOURCE) awkgram.tab.[ch].bak makefile \ awk.1 a.out: awkgram.tab.o $(OFILES) $(CC) $(CFLAGS) awkgram.tab.o $(OFILES) $(ALLOC) -lm $(OFILES): awk.h awkgram.tab.h proto.h awkgram.tab.c awkgram.tab.h: awk.h proto.h awkgram.y $(YACC) $(YFLAGS) awkgram.y proctab.c: maketab ./maketab awkgram.tab.h >proctab.c maketab: awkgram.tab.h maketab.c $(HOSTCC) $(CFLAGS) maketab.c -o maketab bundle: @cp awkgram.tab.h awkgram.tab.h.bak @cp awkgram.tab.c awkgram.tab.c.bak @bundle $(SHIP) tar: @cp awkgram.tab.h awkgram.tab.h.bak @cp awkgram.tab.c awkgram.tab.c.bak @bundle $(SHIP) >awk.shar @tar cf awk.tar $(SHIP) gzip awk.tar ls -l awk.tar.gz @zip awk.zip $(SHIP) ls -l awk.zip gitadd: git add README LICENSE FIXES \ awk.h proto.h awkgram.y lex.c b.c main.c maketab.c parse.c \ lib.c run.c tran.c \ makefile awk.1 testdir gitpush: # only do this once: # git remote add origin https://github.com/onetrueawk/awk.git git push -u origin master names: @echo $(LISTING) test check: ./REGRESS clean: testclean rm -f a.out *.o *.obj maketab maketab.exe *.bb *.bbg *.da *.gcov *.gcno *.gcda # proctab.c cleaner: testclean rm -f a.out *.o *.obj maketab maketab.exe *.bb *.bbg *.da *.gcov *.gcno *.gcda proctab.c awkgram.tab.* # This is a bit of a band-aid until we can invest some more time # in the test suite. testclean: cd testdir; rm -fr arnold-fixes beebe devnull echo foo* \ glop glop1 glop2 lilly.diff tempbig tempsmall time # For the habits of GNU maintainers: distclean: cleaner diff --git a/contrib/one-true-awk/maketab.c b/contrib/one-true-awk/maketab.c index d4b756ad6706..3a80c87725ac 100644 --- a/contrib/one-true-awk/maketab.c +++ b/contrib/one-true-awk/maketab.c @@ -1,193 +1,193 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ /* * this program makes the table to link function names * and type indices that is used by execute() in run.c. * it finds the indices in awkgram.tab.h, produced by bison. */ #include #include #include #include "awk.h" #include "awkgram.tab.h" struct xx { int token; const char *name; const char *pname; } proc[] = { { PROGRAM, "program", NULL }, { BOR, "boolop", " || " }, { AND, "boolop", " && " }, { NOT, "boolop", " !" }, { NE, "relop", " != " }, { EQ, "relop", " == " }, { LE, "relop", " <= " }, { LT, "relop", " < " }, { GE, "relop", " >= " }, { GT, "relop", " > " }, { ARRAY, "array", NULL }, { INDIRECT, "indirect", "$(" }, { SUBSTR, "substr", "substr" }, - { SUB, "sub", "sub" }, - { GSUB, "gsub", "gsub" }, + { SUB, "dosub", "sub" }, + { GSUB, "dosub", "gsub" }, { INDEX, "sindex", "sindex" }, { SPRINTF, "awksprintf", "sprintf " }, { ADD, "arith", " + " }, { MINUS, "arith", " - " }, { MULT, "arith", " * " }, { DIVIDE, "arith", " / " }, { MOD, "arith", " % " }, { UMINUS, "arith", " -" }, { UPLUS, "arith", " +" }, { POWER, "arith", " **" }, { PREINCR, "incrdecr", "++" }, { POSTINCR, "incrdecr", "++" }, { PREDECR, "incrdecr", "--" }, { POSTDECR, "incrdecr", "--" }, { CAT, "cat", " " }, { PASTAT, "pastat", NULL }, { PASTAT2, "dopa2", NULL }, { MATCH, "matchop", " ~ " }, { NOTMATCH, "matchop", " !~ " }, { MATCHFCN, "matchop", "matchop" }, { INTEST, "intest", "intest" }, { PRINTF, "awkprintf", "printf" }, { PRINT, "printstat", "print" }, { CLOSE, "closefile", "closefile" }, { DELETE, "awkdelete", "awkdelete" }, { SPLIT, "split", "split" }, { ASSIGN, "assign", " = " }, { ADDEQ, "assign", " += " }, { SUBEQ, "assign", " -= " }, { MULTEQ, "assign", " *= " }, { DIVEQ, "assign", " /= " }, { MODEQ, "assign", " %= " }, { POWEQ, "assign", " ^= " }, { CONDEXPR, "condexpr", " ?: " }, { IF, "ifstat", "if(" }, { WHILE, "whilestat", "while(" }, { FOR, "forstat", "for(" }, { DO, "dostat", "do" }, { IN, "instat", "instat" }, { NEXT, "jump", "next" }, { NEXTFILE, "jump", "nextfile" }, { EXIT, "jump", "exit" }, { BREAK, "jump", "break" }, { CONTINUE, "jump", "continue" }, { RETURN, "jump", "ret" }, { BLTIN, "bltin", "bltin" }, { CALL, "call", "call" }, { ARG, "arg", "arg" }, { VARNF, "getnf", "NF" }, { GETLINE, "awkgetline", "getline" }, { GENSUB, "gensub", "gensub" }, { 0, "", "" }, }; #define SIZE (LASTTOKEN - FIRSTTOKEN + 1) const char *table[SIZE]; char *names[SIZE]; int main(int argc, char *argv[]) { const struct xx *p; int i, n, tok; char c; FILE *fp; char buf[200], name[200], def[200]; enum { TOK_UNKNOWN, TOK_ENUM, TOK_DEFINE } tokentype = TOK_UNKNOWN; printf("#include \n"); printf("#include \"awk.h\"\n"); printf("#include \"awkgram.tab.h\"\n\n"); if (argc != 2) { fprintf(stderr, "usage: maketab YTAB_H\n"); exit(1); } if ((fp = fopen(argv[1], "r")) == NULL) { fprintf(stderr, "maketab can't open %s!\n", argv[1]); exit(1); } printf("static const char * const printname[%d] = {\n", SIZE); i = 0; while (fgets(buf, sizeof buf, fp) != NULL) { // 199 is sizeof(def) - 1 if (tokentype != TOK_ENUM) { n = sscanf(buf, "%1c %199s %199s %d", &c, def, name, &tok); if (n == 4 && c == '#' && strcmp(def, "define") == 0) { tokentype = TOK_DEFINE; } else if (tokentype != TOK_UNKNOWN) { continue; } } if (tokentype != TOK_DEFINE) { /* not a valid #define, bison uses enums now */ n = sscanf(buf, "%199s = %d,\n", name, &tok); if (n != 2) continue; tokentype = TOK_ENUM; } if (strcmp(name, "YYSTYPE_IS_DECLARED") == 0) { tokentype = TOK_UNKNOWN; continue; } if (tok < FIRSTTOKEN || tok > LASTTOKEN) { tokentype = TOK_UNKNOWN; /* fprintf(stderr, "maketab funny token %d %s ignored\n", tok, buf); */ continue; } names[tok-FIRSTTOKEN] = strdup(name); if (names[tok-FIRSTTOKEN] == NULL) { fprintf(stderr, "maketab out of space copying %s", name); continue; } printf("\t\"%s\",\t/* %d */\n", name, tok); i++; } printf("};\n\n"); for (p=proc; p->token!=0; p++) table[p->token-FIRSTTOKEN] = p->name; printf("\nCell *(*proctab[%d])(Node **, int) = {\n", SIZE); for (i=0; i LASTTOKEN) {\n"); printf("\t\tsnprintf(buf, sizeof(buf), \"token %%d\", n);\n"); printf("\t\treturn buf;\n"); printf("\t}\n"); printf("\treturn printname[n-FIRSTTOKEN];\n"); printf("}\n"); return 0; } diff --git a/contrib/one-true-awk/parse.c b/contrib/one-true-awk/parse.c index 79b8fade83b5..2b7fd1928930 100644 --- a/contrib/one-true-awk/parse.c +++ b/contrib/one-true-awk/parse.c @@ -1,299 +1,299 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ #define DEBUG #include #include #include #include "awk.h" #include "awkgram.tab.h" -Node *nodealloc(int n) +Node *nodealloc(size_t n) { Node *x; x = (Node *) malloc(sizeof(*x) + (n-1) * sizeof(x)); if (x == NULL) FATAL("out of space in nodealloc"); x->nnext = NULL; x->lineno = lineno; return(x); } Node *exptostat(Node *a) { a->ntype = NSTAT; return(a); } Node *node1(int a, Node *b) { Node *x; x = nodealloc(1); x->nobj = a; x->narg[0]=b; return(x); } Node *node2(int a, Node *b, Node *c) { Node *x; x = nodealloc(2); x->nobj = a; x->narg[0] = b; x->narg[1] = c; return(x); } Node *node3(int a, Node *b, Node *c, Node *d) { Node *x; x = nodealloc(3); x->nobj = a; x->narg[0] = b; x->narg[1] = c; x->narg[2] = d; return(x); } Node *node4(int a, Node *b, Node *c, Node *d, Node *e) { Node *x; x = nodealloc(4); x->nobj = a; x->narg[0] = b; x->narg[1] = c; x->narg[2] = d; x->narg[3] = e; return(x); } Node *node5(int a, Node *b, Node *c, Node *d, Node *e, Node *f) { Node *x; x = nodealloc(5); x->nobj = a; x->narg[0] = b; x->narg[1] = c; x->narg[2] = d; x->narg[3] = e; x->narg[4] = f; return(x); } Node *stat1(int a, Node *b) { Node *x; x = node1(a,b); x->ntype = NSTAT; return(x); } Node *stat2(int a, Node *b, Node *c) { Node *x; x = node2(a,b,c); x->ntype = NSTAT; return(x); } Node *stat3(int a, Node *b, Node *c, Node *d) { Node *x; x = node3(a,b,c,d); x->ntype = NSTAT; return(x); } Node *stat4(int a, Node *b, Node *c, Node *d, Node *e) { Node *x; x = node4(a,b,c,d,e); x->ntype = NSTAT; return(x); } Node *op1(int a, Node *b) { Node *x; x = node1(a,b); x->ntype = NEXPR; return(x); } Node *op2(int a, Node *b, Node *c) { Node *x; x = node2(a,b,c); x->ntype = NEXPR; return(x); } Node *op3(int a, Node *b, Node *c, Node *d) { Node *x; x = node3(a,b,c,d); x->ntype = NEXPR; return(x); } Node *op4(int a, Node *b, Node *c, Node *d, Node *e) { Node *x; x = node4(a,b,c,d,e); x->ntype = NEXPR; return(x); } Node *op5(int a, Node *b, Node *c, Node *d, Node *e, Node *f) { Node *x; x = node5(a,b,c,d,e,f); x->ntype = NEXPR; return(x); } Node *celltonode(Cell *a, int b) { Node *x; a->ctype = OCELL; a->csub = b; x = node1(0, (Node *) a); x->ntype = NVALUE; return(x); } Node *rectonode(void) /* make $0 into a Node */ { extern Cell *literal0; return op1(INDIRECT, celltonode(literal0, CUNK)); } Node *makearr(Node *p) { Cell *cp; if (isvalue(p)) { cp = (Cell *) (p->narg[0]); if (isfcn(cp)) SYNTAX( "%s is a function, not an array", cp->nval ); else if (!isarr(cp)) { xfree(cp->sval); cp->sval = (char *) makesymtab(NSYMTAB); cp->tval = ARR; } } return p; } #define PA2NUM 50 /* max number of pat,pat patterns allowed */ int paircnt; /* number of them in use */ int pairstack[PA2NUM]; /* state of each pat,pat */ Node *pa2stat(Node *a, Node *b, Node *c) /* pat, pat {...} */ { Node *x; x = node4(PASTAT2, a, b, c, itonp(paircnt)); if (paircnt++ >= PA2NUM) SYNTAX( "limited to %d pat,pat statements", PA2NUM ); x->ntype = NSTAT; return(x); } Node *linkum(Node *a, Node *b) { Node *c; if (errorflag) /* don't link things that are wrong */ return a; if (a == NULL) return(b); else if (b == NULL) return(a); for (c = a; c->nnext != NULL; c = c->nnext) ; c->nnext = b; return(a); } void defn(Cell *v, Node *vl, Node *st) /* turn on FCN bit in definition, */ { /* body of function, arglist */ Node *p; int n; if (isarr(v)) { SYNTAX( "`%s' is an array name and a function name", v->nval ); return; } if (isarg(v->nval) != -1) { SYNTAX( "`%s' is both function name and argument name", v->nval ); return; } v->tval = FCN; v->sval = (char *) st; n = 0; /* count arguments */ for (p = vl; p; p = p->nnext) n++; v->fval = n; DPRINTF("defining func %s (%d args)\n", v->nval, n); } int isarg(const char *s) /* is s in argument list for current function? */ { /* return -1 if not, otherwise arg # */ extern Node *arglist; Node *p = arglist; int n; for (n = 0; p != NULL; p = p->nnext, n++) if (strcmp(((Cell *)(p->narg[0]))->nval, s) == 0) return n; return -1; } int ptoi(void *p) /* convert pointer to integer */ { return (int) (long) p; /* swearing that p fits, of course */ } Node *itonp(int i) /* and vice versa */ { return (Node *) (long) i; } diff --git a/contrib/one-true-awk/proto.h b/contrib/one-true-awk/proto.h index 5cd3afb8cf85..b44f9e7a5599 100644 --- a/contrib/one-true-awk/proto.h +++ b/contrib/one-true-awk/proto.h @@ -1,209 +1,207 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ extern int yywrap(void); extern void setfname(Cell *); extern int constnode(Node *); extern char *strnode(Node *); extern Node *notnull(Node *); extern int yyparse(void); extern int yylex(void); extern void startreg(void); extern int input(void); extern void unput(int); extern void unputstr(const char *); extern int yylook(void); extern int yyback(int *, int); extern int yyinput(void); extern fa *makedfa(const char *, bool); extern fa *mkdfa(const char *, bool); extern int makeinit(fa *, bool); extern void penter(Node *); extern void freetr(Node *); -extern int hexstr(const uschar **); extern int quoted(const uschar **); -extern char *cclenter(const char *); +extern int *cclenter(const char *); extern noreturn void overflo(const char *); extern void cfoll(fa *, Node *); extern int first(Node *); extern void follow(Node *); -extern int member(int, const char *); +extern int member(int, int *); extern int match(fa *, const char *); extern int pmatch(fa *, const char *); extern int nematch(fa *, const char *); extern bool fnematch(fa *, FILE *, char **, int *, int); extern Node *reparse(const char *); extern Node *regexp(void); extern Node *primary(void); extern Node *concat(Node *); extern Node *alt(Node *); extern Node *unary(Node *); extern int relex(void); extern int cgoto(fa *, int, int); extern void freefa(fa *); extern int pgetc(void); extern char *cursource(void); -extern Node *nodealloc(int); +extern Node *nodealloc(size_t); extern Node *exptostat(Node *); extern Node *node1(int, Node *); extern Node *node2(int, Node *, Node *); extern Node *node3(int, Node *, Node *, Node *); extern Node *node4(int, Node *, Node *, Node *, Node *); extern Node *node5(int, Node *, Node *, Node *, Node *, Node *); extern Node *stat3(int, Node *, Node *, Node *); extern Node *op2(int, Node *, Node *); extern Node *op1(int, Node *); extern Node *stat1(int, Node *); extern Node *op3(int, Node *, Node *, Node *); extern Node *op4(int, Node *, Node *, Node *, Node *); extern Node *op5(int, Node *, Node *, Node *, Node *, Node *); extern Node *stat2(int, Node *, Node *); extern Node *stat4(int, Node *, Node *, Node *, Node *); extern Node *celltonode(Cell *, int); extern Node *rectonode(void); extern Node *makearr(Node *); extern Node *pa2stat(Node *, Node *, Node *); extern Node *linkum(Node *, Node *); extern void defn(Cell *, Node *, Node *); extern int isarg(const char *); extern const char *tokname(int); extern Cell *(*proctab[])(Node **, int); extern int ptoi(void *); extern Node *itonp(int); extern void syminit(void); extern void arginit(int, char **); extern void envinit(char **); extern Array *makesymtab(int); extern void freesymtab(Cell *); extern void freeelem(Cell *, const char *); extern Cell *setsymtab(const char *, const char *, double, unsigned int, Array *); extern int hash(const char *, int); extern void rehash(Array *); extern Cell *lookup(const char *, Array *); extern double setfval(Cell *, double); extern void funnyvar(Cell *, const char *); extern char *setsval(Cell *, const char *); extern double getfval(Cell *); extern char *getsval(Cell *); extern char *getpssval(Cell *); /* for print */ extern char *tostring(const char *); extern char *tostringN(const char *, size_t); extern char *qstring(const char *, int); extern Cell *catstr(Cell *, Cell *); extern void recinit(unsigned int); extern void initgetrec(void); extern void makefields(int, int); extern void growfldtab(int n); extern void savefs(void); extern int getrec(char **, int *, bool); extern void nextfile(void); extern int readrec(char **buf, int *bufsize, FILE *inf, bool isnew); extern char *getargv(int); extern void setclvar(char *); extern void fldbld(void); extern void cleanfld(int, int); extern void newfld(int); extern void setlastfld(int); extern int refldbld(const char *, const char *); extern void recbld(void); extern Cell *fieldadr(int); extern void yyerror(const char *); extern void bracecheck(void); extern void bcheck2(int, int, int); extern void SYNTAX(const char *, ...) __attribute__((__format__(__printf__, 1, 2))); extern noreturn void FATAL(const char *, ...) __attribute__((__format__(__printf__, 1, 2))); extern void WARNING(const char *, ...) __attribute__((__format__(__printf__, 1, 2))); extern void error(void); extern void eprint(void); extern void bclass(int); extern double errcheck(double, const char *); extern int isclvar(const char *); extern bool is_valid_number(const char *s, bool trailing_stuff_ok, bool *no_trailing, double *result); #define is_number(s, val) is_valid_number(s, false, NULL, val) extern int adjbuf(char **pb, int *sz, int min, int q, char **pbp, const char *what); extern void run(Node *); extern Cell *execute(Node *); extern Cell *program(Node **, int); extern Cell *call(Node **, int); extern Cell *copycell(Cell *); extern Cell *arg(Node **, int); extern Cell *jump(Node **, int); extern Cell *awkgetline(Node **, int); extern Cell *getnf(Node **, int); extern Cell *array(Node **, int); extern Cell *awkdelete(Node **, int); extern Cell *intest(Node **, int); extern Cell *matchop(Node **, int); extern Cell *boolop(Node **, int); extern Cell *relop(Node **, int); extern void tfree(Cell *); extern Cell *gettemp(void); extern Cell *field(Node **, int); extern Cell *indirect(Node **, int); extern Cell *substr(Node **, int); extern Cell *sindex(Node **, int); extern int format(char **, int *, const char *, Node *); extern Cell *awksprintf(Node **, int); extern Cell *awkprintf(Node **, int); extern Cell *arith(Node **, int); extern double ipow(double, int); extern Cell *incrdecr(Node **, int); extern Cell *assign(Node **, int); extern Cell *cat(Node **, int); extern Cell *pastat(Node **, int); extern Cell *dopa2(Node **, int); extern Cell *split(Node **, int); extern Cell *condexpr(Node **, int); extern Cell *ifstat(Node **, int); extern Cell *whilestat(Node **, int); extern Cell *dostat(Node **, int); extern Cell *forstat(Node **, int); extern Cell *instat(Node **, int); extern Cell *bltin(Node **, int); extern Cell *printstat(Node **, int); extern Cell *nullproc(Node **, int); extern FILE *redirect(int, Node *); extern FILE *openfile(int, const char *, bool *); extern const char *filename(FILE *); extern Cell *closefile(Node **, int); extern void closeall(void); -extern Cell *sub(Node **, int); -extern Cell *gsub(Node **, int); +extern Cell *dosub(Node **, int); extern Cell *gensub(Node **, int); extern FILE *popen(const char *, const char *); extern int pclose(FILE *); extern const char *flags2str(int flags); diff --git a/contrib/one-true-awk/run.c b/contrib/one-true-awk/run.c index b81dafba35e6..ede3ba7d9827 100644 --- a/contrib/one-true-awk/run.c +++ b/contrib/one-true-awk/run.c @@ -1,2366 +1,2854 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ #define DEBUG #include #include #include -#include #include #include #include #include #include #include #include #include #include #include #include "awk.h" #include "awkgram.tab.h" + static void stdinit(void); static void flush_all(void); +static char *wide_char_to_byte_str(int rune, size_t *outlen); #if 1 #define tempfree(x) do { if (istemp(x)) tfree(x); } while (/*CONSTCOND*/0) #else void tempfree(Cell *p) { if (p->ctype == OCELL && (p->csub < CUNK || p->csub > CFREE)) { WARNING("bad csub %d in Cell %d %s", p->csub, p->ctype, p->sval); } if (istemp(p)) tfree(p); } #endif /* do we really need these? */ /* #ifdef _NFILE */ /* #ifndef FOPEN_MAX */ /* #define FOPEN_MAX _NFILE */ /* #endif */ /* #endif */ /* */ /* #ifndef FOPEN_MAX */ /* #define FOPEN_MAX 40 */ /* max number of open files */ /* #endif */ /* */ /* #ifndef RAND_MAX */ /* #define RAND_MAX 32767 */ /* all that ansi guarantees */ /* #endif */ jmp_buf env; extern int pairstack[]; extern Awkfloat srand_seed; Node *winner = NULL; /* root of parse tree */ Cell *tmps; /* free temporary cells for execution */ static Cell truecell ={ OBOOL, BTRUE, 0, 0, 1.0, NUM, NULL, NULL }; Cell *True = &truecell; static Cell falsecell ={ OBOOL, BFALSE, 0, 0, 0.0, NUM, NULL, NULL }; Cell *False = &falsecell; static Cell breakcell ={ OJUMP, JBREAK, 0, 0, 0.0, NUM, NULL, NULL }; Cell *jbreak = &breakcell; static Cell contcell ={ OJUMP, JCONT, 0, 0, 0.0, NUM, NULL, NULL }; Cell *jcont = &contcell; static Cell nextcell ={ OJUMP, JNEXT, 0, 0, 0.0, NUM, NULL, NULL }; Cell *jnext = &nextcell; static Cell nextfilecell ={ OJUMP, JNEXTFILE, 0, 0, 0.0, NUM, NULL, NULL }; Cell *jnextfile = &nextfilecell; static Cell exitcell ={ OJUMP, JEXIT, 0, 0, 0.0, NUM, NULL, NULL }; Cell *jexit = &exitcell; static Cell retcell ={ OJUMP, JRET, 0, 0, 0.0, NUM, NULL, NULL }; Cell *jret = &retcell; static Cell tempcell ={ OCELL, CTEMP, 0, EMPTY, 0.0, NUM|STR|DONTFREE, NULL, NULL }; Node *curnode = NULL; /* the node being executed, for debugging */ /* buffer memory management */ int adjbuf(char **pbuf, int *psiz, int minlen, int quantum, char **pbptr, const char *whatrtn) /* pbuf: address of pointer to buffer being managed * psiz: address of buffer size variable * minlen: minimum length of buffer needed * quantum: buffer size quantum * pbptr: address of movable pointer into buffer, or 0 if none * whatrtn: name of the calling routine if failure should cause fatal error * * return 0 for realloc failure, !=0 for success */ { if (minlen > *psiz) { char *tbuf; int rminlen = quantum ? minlen % quantum : 0; int boff = pbptr ? *pbptr - *pbuf : 0; /* round up to next multiple of quantum */ if (rminlen) minlen += quantum - rminlen; tbuf = (char *) realloc(*pbuf, minlen); DPRINTF("adjbuf %s: %d %d (pbuf=%p, tbuf=%p)\n", whatrtn, *psiz, minlen, (void*)*pbuf, (void*)tbuf); if (tbuf == NULL) { if (whatrtn) FATAL("out of memory in %s", whatrtn); return 0; } *pbuf = tbuf; *psiz = minlen; if (pbptr) *pbptr = tbuf + boff; } return 1; } void run(Node *a) /* execution of parse tree starts here */ { stdinit(); execute(a); closeall(); } Cell *execute(Node *u) /* execute a node of the parse tree */ { Cell *(*proc)(Node **, int); Cell *x; Node *a; if (u == NULL) return(True); for (a = u; ; a = a->nnext) { curnode = a; if (isvalue(a)) { x = (Cell *) (a->narg[0]); if (isfld(x) && !donefld) fldbld(); else if (isrec(x) && !donerec) recbld(); return(x); } if (notlegal(a->nobj)) /* probably a Cell* but too risky to print */ FATAL("illegal statement"); proc = proctab[a->nobj-FIRSTTOKEN]; x = (*proc)(a->narg, a->nobj); if (isfld(x) && !donefld) fldbld(); else if (isrec(x) && !donerec) recbld(); if (isexpr(a)) return(x); if (isjump(x)) return(x); if (a->nnext == NULL) return(x); tempfree(x); } } Cell *program(Node **a, int n) /* execute an awk program */ { /* a[0] = BEGIN, a[1] = body, a[2] = END */ Cell *x; if (setjmp(env) != 0) goto ex; if (a[0]) { /* BEGIN */ x = execute(a[0]); if (isexit(x)) return(True); if (isjump(x)) FATAL("illegal break, continue, next or nextfile from BEGIN"); tempfree(x); } if (a[1] || a[2]) while (getrec(&record, &recsize, true) > 0) { x = execute(a[1]); if (isexit(x)) break; tempfree(x); } ex: if (setjmp(env) != 0) /* handles exit within END */ goto ex1; if (a[2]) { /* END */ x = execute(a[2]); if (isbreak(x) || isnext(x) || iscont(x)) FATAL("illegal break, continue, next or nextfile from END"); tempfree(x); } ex1: return(True); } struct Frame { /* stack frame for awk function calls */ int nargs; /* number of arguments in this call */ Cell *fcncell; /* pointer to Cell for function */ Cell **args; /* pointer to array of arguments after execute */ Cell *retval; /* return value */ }; #define NARGS 50 /* max args in a call */ struct Frame *frame = NULL; /* base of stack frames; dynamically allocated */ int nframe = 0; /* number of frames allocated */ struct Frame *frp = NULL; /* frame pointer. bottom level unused */ Cell *call(Node **a, int n) /* function call. very kludgy and fragile */ { static const Cell newcopycell = { OCELL, CCOPY, 0, EMPTY, 0.0, NUM|STR|DONTFREE, NULL, NULL }; int i, ncall, ndef; int freed = 0; /* handles potential double freeing when fcn & param share a tempcell */ Node *x; Cell *args[NARGS], *oargs[NARGS]; /* BUG: fixed size arrays */ Cell *y, *z, *fcn; char *s; fcn = execute(a[0]); /* the function itself */ s = fcn->nval; if (!isfcn(fcn)) FATAL("calling undefined function %s", s); if (frame == NULL) { frp = frame = (struct Frame *) calloc(nframe += 100, sizeof(*frame)); if (frame == NULL) FATAL("out of space for stack frames calling %s", s); } for (ncall = 0, x = a[1]; x != NULL; x = x->nnext) /* args in call */ ncall++; ndef = (int) fcn->fval; /* args in defn */ DPRINTF("calling %s, %d args (%d in defn), frp=%d\n", s, ncall, ndef, (int) (frp-frame)); if (ncall > ndef) WARNING("function %s called with %d args, uses only %d", s, ncall, ndef); if (ncall + ndef > NARGS) FATAL("function %s has %d arguments, limit %d", s, ncall+ndef, NARGS); for (i = 0, x = a[1]; x != NULL; i++, x = x->nnext) { /* get call args */ DPRINTF("evaluate args[%d], frp=%d:\n", i, (int) (frp-frame)); y = execute(x); oargs[i] = y; DPRINTF("args[%d]: %s %f <%s>, t=%o\n", i, NN(y->nval), y->fval, isarr(y) ? "(array)" : NN(y->sval), y->tval); if (isfcn(y)) FATAL("can't use function %s as argument in %s", y->nval, s); if (isarr(y)) args[i] = y; /* arrays by ref */ else args[i] = copycell(y); tempfree(y); } for ( ; i < ndef; i++) { /* add null args for ones not provided */ args[i] = gettemp(); *args[i] = newcopycell; } frp++; /* now ok to up frame */ if (frp >= frame + nframe) { int dfp = frp - frame; /* old index */ frame = (struct Frame *) realloc(frame, (nframe += 100) * sizeof(*frame)); if (frame == NULL) FATAL("out of space for stack frames in %s", s); frp = frame + dfp; } frp->fcncell = fcn; frp->args = args; frp->nargs = ndef; /* number defined with (excess are locals) */ frp->retval = gettemp(); DPRINTF("start exec of %s, frp=%d\n", s, (int) (frp-frame)); y = execute((Node *)(fcn->sval)); /* execute body */ DPRINTF("finished exec of %s, frp=%d\n", s, (int) (frp-frame)); for (i = 0; i < ndef; i++) { Cell *t = frp->args[i]; if (isarr(t)) { if (t->csub == CCOPY) { if (i >= ncall) { freesymtab(t); t->csub = CTEMP; tempfree(t); } else { oargs[i]->tval = t->tval; oargs[i]->tval &= ~(STR|NUM|DONTFREE); oargs[i]->sval = t->sval; tempfree(t); } } } else if (t != y) { /* kludge to prevent freeing twice */ t->csub = CTEMP; tempfree(t); } else if (t == y && t->csub == CCOPY) { t->csub = CTEMP; tempfree(t); freed = 1; } } tempfree(fcn); if (isexit(y) || isnext(y)) return y; if (freed == 0) { tempfree(y); /* don't free twice! */ } z = frp->retval; /* return value */ DPRINTF("%s returns %g |%s| %o\n", s, getfval(z), getsval(z), z->tval); frp--; return(z); } Cell *copycell(Cell *x) /* make a copy of a cell in a temp */ { Cell *y; /* copy is not constant or field */ y = gettemp(); y->tval = x->tval & ~(CON|FLD|REC); y->csub = CCOPY; /* prevents freeing until call is over */ y->nval = x->nval; /* BUG? */ if (isstr(x) /* || x->ctype == OCELL */) { y->sval = tostring(x->sval); y->tval &= ~DONTFREE; } else y->tval |= DONTFREE; y->fval = x->fval; return y; } Cell *arg(Node **a, int n) /* nth argument of a function */ { n = ptoi(a[0]); /* argument number, counting from 0 */ DPRINTF("arg(%d), frp->nargs=%d\n", n, frp->nargs); if (n+1 > frp->nargs) FATAL("argument #%d of function %s was not supplied", n+1, frp->fcncell->nval); return frp->args[n]; } Cell *jump(Node **a, int n) /* break, continue, next, nextfile, return */ { Cell *y; switch (n) { case EXIT: if (a[0] != NULL) { y = execute(a[0]); errorflag = (int) getfval(y); tempfree(y); } longjmp(env, 1); case RETURN: if (a[0] != NULL) { y = execute(a[0]); if ((y->tval & (STR|NUM)) == (STR|NUM)) { setsval(frp->retval, getsval(y)); frp->retval->fval = getfval(y); frp->retval->tval |= NUM; } else if (y->tval & STR) setsval(frp->retval, getsval(y)); else if (y->tval & NUM) setfval(frp->retval, getfval(y)); else /* can't happen */ FATAL("bad type variable %d", y->tval); tempfree(y); } return(jret); case NEXT: return(jnext); case NEXTFILE: nextfile(); return(jnextfile); case BREAK: return(jbreak); case CONTINUE: return(jcont); default: /* can't happen */ FATAL("illegal jump type %d", n); } return 0; /* not reached */ } Cell *awkgetline(Node **a, int n) /* get next line from specific input */ { /* a[0] is variable, a[1] is operator, a[2] is filename */ Cell *r, *x; extern Cell **fldtab; FILE *fp; char *buf; int bufsize = recsize; int mode; bool newflag; double result; if ((buf = (char *) malloc(bufsize)) == NULL) FATAL("out of memory in getline"); fflush(stdout); /* in case someone is waiting for a prompt */ r = gettemp(); if (a[1] != NULL) { /* getline < file */ x = execute(a[2]); /* filename */ mode = ptoi(a[1]); if (mode == '|') /* input pipe */ mode = LE; /* arbitrary flag */ fp = openfile(mode, getsval(x), &newflag); tempfree(x); if (fp == NULL) n = -1; else n = readrec(&buf, &bufsize, fp, newflag); if (n <= 0) { ; } else if (a[0] != NULL) { /* getline var sval, & result)) { x->fval = result; x->tval |= NUM; } tempfree(x); } else { /* getline sval, & result)) { fldtab[0]->fval = result; fldtab[0]->tval |= NUM; } } } else { /* bare getline; use current input */ if (a[0] == NULL) /* getline */ n = getrec(&record, &recsize, true); else { /* getline var */ n = getrec(&buf, &bufsize, false); - x = execute(a[0]); - setsval(x, buf); - if (is_number(x->sval, & result)) { - x->fval = result; - x->tval |= NUM; + if (n > 0) { + x = execute(a[0]); + setsval(x, buf); + if (is_number(x->sval, & result)) { + x->fval = result; + x->tval |= NUM; + } + tempfree(x); } - tempfree(x); } } setfval(r, (Awkfloat) n); free(buf); return r; } Cell *getnf(Node **a, int n) /* get NF */ { if (!donefld) fldbld(); return (Cell *) a[0]; } static char * makearraystring(Node *p, const char *func) { char *buf; int bufsz = recsize; size_t blen; if ((buf = (char *) malloc(bufsz)) == NULL) { FATAL("%s: out of memory", func); } blen = 0; buf[blen] = '\0'; for (; p; p = p->nnext) { Cell *x = execute(p); /* expr */ char *s = getsval(x); size_t seplen = strlen(getsval(subseploc)); size_t nsub = p->nnext ? seplen : 0; size_t slen = strlen(s); size_t tlen = blen + slen + nsub; if (!adjbuf(&buf, &bufsz, tlen + 1, recsize, 0, func)) { FATAL("%s: out of memory %s[%s...]", func, x->nval, buf); } memcpy(buf + blen, s, slen); if (nsub) { memcpy(buf + blen + slen, *SUBSEP, nsub); } buf[tlen] = '\0'; blen = tlen; tempfree(x); } return buf; } Cell *array(Node **a, int n) /* a[0] is symtab, a[1] is list of subscripts */ { Cell *x, *z; char *buf; x = execute(a[0]); /* Cell* for symbol table */ buf = makearraystring(a[1], __func__); if (!isarr(x)) { DPRINTF("making %s into an array\n", NN(x->nval)); if (freeable(x)) xfree(x->sval); x->tval &= ~(STR|NUM|DONTFREE); x->tval |= ARR; x->sval = (char *) makesymtab(NSYMTAB); } z = setsymtab(buf, "", 0.0, STR|NUM, (Array *) x->sval); z->ctype = OCELL; z->csub = CVAR; tempfree(x); free(buf); return(z); } Cell *awkdelete(Node **a, int n) /* a[0] is symtab, a[1] is list of subscripts */ { Cell *x; x = execute(a[0]); /* Cell* for symbol table */ if (x == symtabloc) { FATAL("cannot delete SYMTAB or its elements"); } if (!isarr(x)) return True; if (a[1] == NULL) { /* delete the elements, not the table */ freesymtab(x); x->tval &= ~STR; x->tval |= ARR; x->sval = (char *) makesymtab(NSYMTAB); } else { char *buf = makearraystring(a[1], __func__); freeelem(x, buf); free(buf); } tempfree(x); return True; } Cell *intest(Node **a, int n) /* a[0] is index (list), a[1] is symtab */ { Cell *ap, *k; char *buf; ap = execute(a[1]); /* array name */ if (!isarr(ap)) { DPRINTF("making %s into an array\n", ap->nval); if (freeable(ap)) xfree(ap->sval); ap->tval &= ~(STR|NUM|DONTFREE); ap->tval |= ARR; ap->sval = (char *) makesymtab(NSYMTAB); } buf = makearraystring(a[0], __func__); k = lookup(buf, (Array *) ap->sval); tempfree(ap); free(buf); if (k == NULL) return(False); else return(True); } +/* ======== utf-8 code ========== */ + +/* + * Awk strings can contain ascii, random 8-bit items (eg Latin-1), + * or utf-8. u8_isutf tests whether a string starts with a valid + * utf-8 sequence, and returns 0 if not (e.g., high bit set). + * u8_nextlen returns length of next valid sequence, which is + * 1 for ascii, 2..4 for utf-8, or 1 for high bit non-utf. + * u8_strlen returns length of string in valid utf-8 sequences + * and/or high-bit bytes. Conversion functions go between byte + * number and character number. + * + * In theory, this behaves the same as before for non-utf8 bytes. + * + * Limited checking! This is a potential security hole. + */ + +/* is s the beginning of a valid utf-8 string? */ +/* return length 1..4 if yes, 0 if no */ +int u8_isutf(const char *s) +{ + int n, ret; + unsigned char c; + + c = s[0]; + if (c < 128 || awk_mb_cur_max == 1) + return 1; /* what if it's 0? */ + + n = strlen(s); + if (n >= 2 && ((c>>5) & 0x7) == 0x6 && (s[1] & 0xC0) == 0x80) { + ret = 2; /* 110xxxxx 10xxxxxx */ + } else if (n >= 3 && ((c>>4) & 0xF) == 0xE && (s[1] & 0xC0) == 0x80 + && (s[2] & 0xC0) == 0x80) { + ret = 3; /* 1110xxxx 10xxxxxx 10xxxxxx */ + } else if (n >= 4 && ((c>>3) & 0x1F) == 0x1E && (s[1] & 0xC0) == 0x80 + && (s[2] & 0xC0) == 0x80 && (s[3] & 0xC0) == 0x80) { + ret = 4; /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ + } else { + ret = 0; + } + return ret; +} + +/* Convert (prefix of) utf8 string to utf-32 rune. */ +/* Sets *rune to the value, returns the length. */ +/* No error checking: watch out. */ +int u8_rune(int *rune, const char *s) +{ + int n, ret; + unsigned char c; + + c = s[0]; + if (c < 128 || awk_mb_cur_max == 1) { + *rune = c; + return 1; + } + + n = strlen(s); + if (n >= 2 && ((c>>5) & 0x7) == 0x6 && (s[1] & 0xC0) == 0x80) { + *rune = ((c & 0x1F) << 6) | (s[1] & 0x3F); /* 110xxxxx 10xxxxxx */ + ret = 2; + } else if (n >= 3 && ((c>>4) & 0xF) == 0xE && (s[1] & 0xC0) == 0x80 + && (s[2] & 0xC0) == 0x80) { + *rune = ((c & 0xF) << 12) | ((s[1] & 0x3F) << 6) | (s[2] & 0x3F); + /* 1110xxxx 10xxxxxx 10xxxxxx */ + ret = 3; + } else if (n >= 4 && ((c>>3) & 0x1F) == 0x1E && (s[1] & 0xC0) == 0x80 + && (s[2] & 0xC0) == 0x80 && (s[3] & 0xC0) == 0x80) { + *rune = ((c & 0x7) << 18) | ((s[1] & 0x3F) << 12) | ((s[2] & 0x3F) << 6) | (s[3] & 0x3F); + /* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ + ret = 4; + } else { + *rune = c; + ret = 1; + } + return ret; /* returns one byte if sequence doesn't look like utf */ +} + +/* return length of next sequence: 1 for ascii or random, 2..4 for valid utf8 */ +int u8_nextlen(const char *s) +{ + int len; + + len = u8_isutf(s); + if (len == 0) + len = 1; + return len; +} + +/* return number of utf characters or single non-utf bytes */ +int u8_strlen(const char *s) +{ + int i, len, n, totlen; + unsigned char c; + + n = strlen(s); + totlen = 0; + for (i = 0; i < n; i += len) { + c = s[i]; + if (c < 128 || awk_mb_cur_max == 1) { + len = 1; + } else { + len = u8_nextlen(&s[i]); + } + totlen++; + if (i > n) + FATAL("bad utf count [%s] n=%d i=%d\n", s, n, i); + } + return totlen; +} + +/* convert utf-8 char number in a string to its byte offset */ +int u8_char2byte(const char *s, int charnum) +{ + int n; + int bytenum = 0; + + while (charnum > 0) { + n = u8_nextlen(s); + s += n; + bytenum += n; + charnum--; + } + return bytenum; +} + +/* convert byte offset in s to utf-8 char number that starts there */ +int u8_byte2char(const char *s, int bytenum) +{ + int i, len, b; + int charnum = 0; /* BUG: what origin? */ + /* should be 0 to match start==0 which means no match */ + + b = strlen(s); + if (bytenum > b) { + return -1; /* ??? */ + } + for (i = 0; i <= bytenum; i += len) { + len = u8_nextlen(s+i); + charnum++; + } + return charnum; +} + +/* runetochar() adapted from rune.c in the Plan 9 distributione */ + +enum +{ + Runeerror = 128, /* from somewhere else */ + Runemax = 0x10FFFF, + + Bit1 = 7, + Bitx = 6, + Bit2 = 5, + Bit3 = 4, + Bit4 = 3, + Bit5 = 2, + + T1 = ((1<<(Bit1+1))-1) ^ 0xFF, /* 0000 0000 */ + Tx = ((1<<(Bitx+1))-1) ^ 0xFF, /* 1000 0000 */ + T2 = ((1<<(Bit2+1))-1) ^ 0xFF, /* 1100 0000 */ + T3 = ((1<<(Bit3+1))-1) ^ 0xFF, /* 1110 0000 */ + T4 = ((1<<(Bit4+1))-1) ^ 0xFF, /* 1111 0000 */ + T5 = ((1<<(Bit5+1))-1) ^ 0xFF, /* 1111 1000 */ + + Rune1 = (1<<(Bit1+0*Bitx))-1, /* 0000 0000 0000 0000 0111 1111 */ + Rune2 = (1<<(Bit2+1*Bitx))-1, /* 0000 0000 0000 0111 1111 1111 */ + Rune3 = (1<<(Bit3+2*Bitx))-1, /* 0000 0000 1111 1111 1111 1111 */ + Rune4 = (1<<(Bit4+3*Bitx))-1, /* 0011 1111 1111 1111 1111 1111 */ + + Maskx = (1< 00-7F */ + if (c <= Rune1) { + str[0] = c; + return 1; + } + + /* two character sequence 00080-007FF => T2 Tx */ + if (c <= Rune2) { + str[0] = T2 | (c >> 1*Bitx); + str[1] = Tx | (c & Maskx); + return 2; + } + + /* three character sequence 00800-0FFFF => T3 Tx Tx */ + if (c > Runemax) + c = Runeerror; + if (c <= Rune3) { + str[0] = T3 | (c >> 2*Bitx); + str[1] = Tx | ((c >> 1*Bitx) & Maskx); + str[2] = Tx | (c & Maskx); + return 3; + } + + /* four character sequence 010000-1FFFFF => T4 Tx Tx Tx */ + str[0] = T4 | (c >> 3*Bitx); + str[1] = Tx | ((c >> 2*Bitx) & Maskx); + str[2] = Tx | ((c >> 1*Bitx) & Maskx); + str[3] = Tx | (c & Maskx); + return 4; +} + + +/* ========== end of utf8 code =========== */ + + + Cell *matchop(Node **a, int n) /* ~ and match() */ { - Cell *x, *y; + Cell *x, *y, *z; char *s, *t; int i; + int cstart, cpatlen, len; fa *pfa; int (*mf)(fa *, const char *) = match, mode = 0; if (n == MATCHFCN) { mf = pmatch; mode = 1; } x = execute(a[1]); /* a[1] = target text */ s = getsval(x); if (a[0] == NULL) /* a[1] == 0: already-compiled reg expr */ i = (*mf)((fa *) a[2], s); else { y = execute(a[2]); /* a[2] = regular expr */ t = getsval(y); pfa = makedfa(t, mode); i = (*mf)(pfa, s); tempfree(y); } - tempfree(x); + z = x; if (n == MATCHFCN) { - int start = patbeg - s + 1; - if (patlen < 0) - start = 0; + int start = patbeg - s + 1; /* origin 1 */ + if (patlen < 0) { + start = 0; /* not found */ + } else { + cstart = u8_byte2char(s, start-1); + cpatlen = 0; + for (i = 0; i < patlen; i += len) { + len = u8_nextlen(patbeg+i); + cpatlen++; + } + + start = cstart; + patlen = cpatlen; + } + setfval(rstartloc, (Awkfloat) start); setfval(rlengthloc, (Awkfloat) patlen); x = gettemp(); x->tval = NUM; x->fval = start; - return x; } else if ((n == MATCH && i == 1) || (n == NOTMATCH && i == 0)) - return(True); + x = True; else - return(False); + x = False; + + tempfree(z); + return x; } Cell *boolop(Node **a, int n) /* a[0] || a[1], a[0] && a[1], !a[0] */ { Cell *x, *y; int i; x = execute(a[0]); i = istrue(x); tempfree(x); switch (n) { case BOR: if (i) return(True); y = execute(a[1]); i = istrue(y); tempfree(y); if (i) return(True); else return(False); case AND: if ( !i ) return(False); y = execute(a[1]); i = istrue(y); tempfree(y); if (i) return(True); else return(False); case NOT: if (i) return(False); else return(True); default: /* can't happen */ FATAL("unknown boolean operator %d", n); } return 0; /*NOTREACHED*/ } Cell *relop(Node **a, int n) /* a[0 < a[1], etc. */ { int i; Cell *x, *y; Awkfloat j; + bool x_is_nan, y_is_nan; x = execute(a[0]); y = execute(a[1]); + x_is_nan = isnan(x->fval); + y_is_nan = isnan(y->fval); if (x->tval&NUM && y->tval&NUM) { + if ((x_is_nan || y_is_nan) && n != NE) + return(False); j = x->fval - y->fval; i = j<0? -1: (j>0? 1: 0); } else { i = strcmp(getsval(x), getsval(y)); } tempfree(x); tempfree(y); switch (n) { case LT: if (i<0) return(True); else return(False); case LE: if (i<=0) return(True); else return(False); - case NE: if (i!=0) return(True); + case NE: if (x_is_nan && y_is_nan) return(True); + else if (i!=0) return(True); else return(False); case EQ: if (i == 0) return(True); else return(False); case GE: if (i>=0) return(True); else return(False); case GT: if (i>0) return(True); else return(False); default: /* can't happen */ FATAL("unknown relational operator %d", n); } return 0; /*NOTREACHED*/ } void tfree(Cell *a) /* free a tempcell */ { if (freeable(a)) { DPRINTF("freeing %s %s %o\n", NN(a->nval), NN(a->sval), a->tval); xfree(a->sval); } if (a == tmps) FATAL("tempcell list is curdled"); a->cnext = tmps; tmps = a; } Cell *gettemp(void) /* get a tempcell */ { int i; Cell *x; if (!tmps) { tmps = (Cell *) calloc(100, sizeof(*tmps)); if (!tmps) FATAL("out of space for temporaries"); for (i = 1; i < 100; i++) tmps[i-1].cnext = &tmps[i]; tmps[i-1].cnext = NULL; } x = tmps; tmps = x->cnext; *x = tempcell; return(x); } Cell *indirect(Node **a, int n) /* $( a[0] ) */ { Awkfloat val; Cell *x; int m; char *s; x = execute(a[0]); val = getfval(x); /* freebsd: defend against super large field numbers */ if ((Awkfloat)INT_MAX < val) FATAL("trying to access out of range field %s", x->nval); m = (int) val; if (m == 0 && !is_number(s = getsval(x), NULL)) /* suspicion! */ FATAL("illegal field $(%s), name \"%s\"", s, x->nval); /* BUG: can x->nval ever be null??? */ tempfree(x); x = fieldadr(m); x->ctype = OCELL; /* BUG? why are these needed? */ x->csub = CFLD; return(x); } Cell *substr(Node **a, int nnn) /* substr(a[0], a[1], a[2]) */ { int k, m, n; + int mb, nb; char *s; int temp; Cell *x, *y, *z = NULL; x = execute(a[0]); y = execute(a[1]); if (a[2] != NULL) z = execute(a[2]); s = getsval(x); - k = strlen(s) + 1; + k = u8_strlen(s) + 1; if (k <= 1) { tempfree(x); tempfree(y); if (a[2] != NULL) { tempfree(z); } x = gettemp(); setsval(x, ""); return(x); } m = (int) getfval(y); if (m <= 0) m = 1; else if (m > k) m = k; tempfree(y); if (a[2] != NULL) { n = (int) getfval(z); tempfree(z); } else n = k - 1; if (n < 0) n = 0; else if (n > k - m) n = k - m; + /* m is start, n is length from there */ DPRINTF("substr: m=%d, n=%d, s=%s\n", m, n, s); y = gettemp(); - temp = s[n+m-1]; /* with thanks to John Linderman */ - s[n+m-1] = '\0'; - setsval(y, s + m - 1); - s[n+m-1] = temp; + mb = u8_char2byte(s, m-1); /* byte offset of start char in s */ + nb = u8_char2byte(s, m-1+n); /* byte offset of end+1 char in s */ + + temp = s[nb]; /* with thanks to John Linderman */ + s[nb] = '\0'; + setsval(y, s + mb); + s[nb] = temp; tempfree(x); return(y); } Cell *sindex(Node **a, int nnn) /* index(a[0], a[1]) */ { Cell *x, *y, *z; char *s1, *s2, *p1, *p2, *q; Awkfloat v = 0.0; x = execute(a[0]); s1 = getsval(x); y = execute(a[1]); s2 = getsval(y); z = gettemp(); for (p1 = s1; *p1 != '\0'; p1++) { for (q = p1, p2 = s2; *p2 != '\0' && *q == *p2; q++, p2++) continue; if (*p2 == '\0') { - v = (Awkfloat) (p1 - s1 + 1); /* origin 1 */ + /* v = (Awkfloat) (p1 - s1 + 1); origin 1 */ + + /* should be a function: used in match() as well */ + int i, len; + v = 0; + for (i = 0; i < p1-s1+1; i += len) { + len = u8_nextlen(s1+i); + v++; + } break; } } tempfree(x); tempfree(y); setfval(z, v); return(z); } +int has_utf8(char *s) /* return 1 if s contains any utf-8 (2 bytes or more) character */ +{ + int n; + + for (n = 0; *s != 0; s += n) { + n = u8_nextlen(s); + if (n > 1) + return 1; + } + return 0; +} + #define MAXNUMSIZE 50 int format(char **pbuf, int *pbufsize, const char *s, Node *a) /* printf-like conversions */ { char *fmt; char *p, *t; const char *os; Cell *x; int flag = 0, n; int fmtwd; /* format width */ int fmtsz = recsize; char *buf = *pbuf; int bufsize = *pbufsize; #define FMTSZ(a) (fmtsz - ((a) - fmt)) #define BUFSZ(a) (bufsize - ((a) - buf)) static bool first = true; static bool have_a_format = false; if (first) { char xbuf[100]; snprintf(xbuf, sizeof(xbuf), "%a", 42.0); have_a_format = (strcmp(xbuf, "0x1.5p+5") == 0); first = false; } os = s; p = buf; if ((fmt = (char *) malloc(fmtsz)) == NULL) FATAL("out of memory in format()"); while (*s) { adjbuf(&buf, &bufsize, MAXNUMSIZE+1+p-buf, recsize, &p, "format1"); if (*s != '%') { *p++ = *s++; continue; } if (*(s+1) == '%') { *p++ = '%'; s += 2; continue; } - /* have to be real careful in case this is a huge number, eg, %100000d */ fmtwd = atoi(s+1); if (fmtwd < 0) fmtwd = -fmtwd; adjbuf(&buf, &bufsize, fmtwd+1+p-buf, recsize, &p, "format2"); for (t = fmt; (*t++ = *s) != '\0'; s++) { if (!adjbuf(&fmt, &fmtsz, MAXNUMSIZE+1+t-fmt, recsize, &t, "format3")) FATAL("format item %.30s... ran format() out of memory", os); /* Ignore size specifiers */ if (strchr("hjLlqtz", *s) != NULL) { /* the ansi panoply */ t--; continue; } if (isalpha((uschar)*s)) break; if (*s == '$') { FATAL("'$' not permitted in awk formats"); } if (*s == '*') { if (a == NULL) { FATAL("not enough args in printf(%s)", os); } x = execute(a); a = a->nnext; snprintf(t - 1, FMTSZ(t - 1), "%d", fmtwd=(int) getfval(x)); if (fmtwd < 0) fmtwd = -fmtwd; adjbuf(&buf, &bufsize, fmtwd+1+p-buf, recsize, &p, "format"); t = fmt + strlen(fmt); tempfree(x); } } *t = '\0'; if (fmtwd < 0) fmtwd = -fmtwd; adjbuf(&buf, &bufsize, fmtwd+1+p-buf, recsize, &p, "format4"); switch (*s) { case 'a': case 'A': if (have_a_format) flag = *s; else flag = 'f'; break; case 'f': case 'e': case 'g': case 'E': case 'G': flag = 'f'; break; case 'd': case 'i': case 'o': case 'x': case 'X': case 'u': flag = (*s == 'd' || *s == 'i') ? 'd' : 'u'; *(t-1) = 'j'; *t = *s; *++t = '\0'; break; case 's': flag = 's'; break; case 'c': flag = 'c'; break; - case '\0': - FATAL("missing printf conversion specifier"); - break; default: WARNING("weird printf conversion %s", fmt); flag = '?'; break; } if (a == NULL) FATAL("not enough args in printf(%s)", os); x = execute(a); a = a->nnext; n = MAXNUMSIZE; if (fmtwd > n) n = fmtwd; adjbuf(&buf, &bufsize, 1+n+p-buf, recsize, &p, "format5"); switch (flag) { - case '?': snprintf(p, BUFSZ(p), "%s", fmt); /* unknown, so dump it too */ + case '?': + snprintf(p, BUFSZ(p), "%s", fmt); /* unknown, so dump it too */ t = getsval(x); n = strlen(t); if (fmtwd > n) n = fmtwd; adjbuf(&buf, &bufsize, 1+strlen(p)+n+p-buf, recsize, &p, "format6"); p += strlen(p); snprintf(p, BUFSZ(p), "%s", t); break; case 'a': case 'A': case 'f': snprintf(p, BUFSZ(p), fmt, getfval(x)); break; case 'd': snprintf(p, BUFSZ(p), fmt, (intmax_t) getfval(x)); break; case 'u': snprintf(p, BUFSZ(p), fmt, (uintmax_t) getfval(x)); break; - case 's': + + case 's': { t = getsval(x); n = strlen(t); - if (fmtwd > n) - n = fmtwd; - if (!adjbuf(&buf, &bufsize, 1+n+p-buf, recsize, &p, "format7")) - FATAL("huge string/format (%d chars) in printf %.30s... ran format() out of memory", n, t); - snprintf(p, BUFSZ(p), fmt, t); + /* if simple format or no utf-8 in the string, sprintf works */ + if (!has_utf8(t) || strcmp(fmt,"%s") == 0) { + if (fmtwd > n) + n = fmtwd; + if (!adjbuf(&buf, &bufsize, 1+n+p-buf, recsize, &p, "format7")) + FATAL("huge string/format (%d chars) in printf %.30s..." \ + " ran format() out of memory", n, t); + snprintf(p, BUFSZ(p), fmt, t); + break; + } + + /* get here if string has utf-8 chars and fmt is not plain %s */ + /* "%-w.ps", where -, w and .p are all optional */ + /* '0' before the w is a flag character */ + /* fmt points at % */ + int ljust = 0, wid = 0, prec = n, pad = 0; + char *f = fmt+1; + if (f[0] == '-') { + ljust = 1; + f++; + } + // flags '0' and '+' are recognized but skipped + if (f[0] == '0') { + f++; + if (f[0] == '+') + f++; + } + if (f[0] == '+') { + f++; + if (f[0] == '0') + f++; + } + if (isdigit(f[0])) { /* there is a wid */ + wid = strtol(f, &f, 10); + } + if (f[0] == '.') { /* there is a .prec */ + prec = strtol(++f, &f, 10); + } + if (prec > u8_strlen(t)) + prec = u8_strlen(t); + pad = wid>prec ? wid - prec : 0; // has to be >= 0 + int i, k, n; + + if (ljust) { // print prec chars from t, then pad blanks + n = u8_char2byte(t, prec); + for (k = 0; k < n; k++) { + //putchar(t[k]); + *p++ = t[k]; + } + for (i = 0; i < pad; i++) { + //printf(" "); + *p++ = ' '; + } + } else { // print pad blanks, then prec chars from t + for (i = 0; i < pad; i++) { + //printf(" "); + *p++ = ' '; + } + n = u8_char2byte(t, prec); + for (k = 0; k < n; k++) { + //putchar(t[k]); + *p++ = t[k]; + } + } + *p = 0; break; - case 'c': + } + + case 'c': { + /* + * If a numeric value is given, awk should just turn + * it into a character and print it: + * BEGIN { printf("%c\n", 65) } + * prints "A". + * + * But what if the numeric value is > 128 and + * represents a valid Unicode code point?!? We do + * our best to convert it back into UTF-8. If we + * can't, we output the encoding of the Unicode + * "invalid character", 0xFFFD. + */ if (isnum(x)) { - if ((int)getfval(x)) - snprintf(p, BUFSZ(p), fmt, (int) getfval(x)); - else { + int charval = (int) getfval(x); + + if (charval != 0) { + if (charval < 128 || awk_mb_cur_max == 1) + snprintf(p, BUFSZ(p), fmt, charval); + else { + // possible unicode character + size_t count; + char *bs = wide_char_to_byte_str(charval, &count); + + if (bs == NULL) { // invalid character + // use unicode invalid character, 0xFFFD + static char invalid_char[] = "\357\277\275"; + bs = invalid_char; + count = 3; + } + t = bs; + n = count; + goto format_percent_c; + } + } else { *p++ = '\0'; /* explicit null byte */ *p = '\0'; /* next output will start here */ } - } else + break; + } + t = getsval(x); + n = u8_nextlen(t); + format_percent_c: + if (n < 2) { /* not utf8 */ snprintf(p, BUFSZ(p), fmt, getsval(x)[0]); + break; + } + + // utf8 character, almost same song and dance as for %s + int ljust = 0, wid = 0, prec = n, pad = 0; + char *f = fmt+1; + if (f[0] == '-') { + ljust = 1; + f++; + } + // flags '0' and '+' are recognized but skipped + if (f[0] == '0') { + f++; + if (f[0] == '+') + f++; + } + if (f[0] == '+') { + f++; + if (f[0] == '0') + f++; + } + if (isdigit(f[0])) { /* there is a wid */ + wid = strtol(f, &f, 10); + } + if (f[0] == '.') { /* there is a .prec */ + prec = strtol(++f, &f, 10); + } + if (prec > 1) // %c --> only one character + prec = 1; + pad = wid>prec ? wid - prec : 0; // has to be >= 0 + int i; + + if (ljust) { // print one char from t, then pad blanks + for (i = 0; i < n; i++) + *p++ = t[i]; + for (i = 0; i < pad; i++) { + //printf(" "); + *p++ = ' '; + } + } else { // print pad blanks, then prec chars from t + for (i = 0; i < pad; i++) { + //printf(" "); + *p++ = ' '; + } + for (i = 0; i < n; i++) + *p++ = t[i]; + } + *p = 0; break; + } default: FATAL("can't happen: bad conversion %c in format()", flag); } + tempfree(x); p += strlen(p); s++; } *p = '\0'; free(fmt); - for ( ; a; a = a->nnext) /* evaluate any remaining args */ - execute(a); + for ( ; a; a = a->nnext) { /* evaluate any remaining args */ + x = execute(a); + tempfree(x); + } *pbuf = buf; *pbufsize = bufsize; return p - buf; } Cell *awksprintf(Node **a, int n) /* sprintf(a[0]) */ { Cell *x; Node *y; char *buf; int bufsz=3*recsize; if ((buf = (char *) malloc(bufsz)) == NULL) FATAL("out of memory in awksprintf"); y = a[0]->nnext; x = execute(a[0]); if (format(&buf, &bufsz, getsval(x), y) == -1) FATAL("sprintf string %.30s... too long. can't happen.", buf); tempfree(x); x = gettemp(); x->sval = buf; x->tval = STR; return(x); } Cell *awkprintf(Node **a, int n) /* printf */ { /* a[0] is list of args, starting with format string */ /* a[1] is redirection operator, a[2] is redirection file */ FILE *fp; Cell *x; Node *y; char *buf; int len; int bufsz=3*recsize; if ((buf = (char *) malloc(bufsz)) == NULL) FATAL("out of memory in awkprintf"); y = a[0]->nnext; x = execute(a[0]); if ((len = format(&buf, &bufsz, getsval(x), y)) == -1) FATAL("printf string %.30s... too long. can't happen.", buf); tempfree(x); if (a[1] == NULL) { /* fputs(buf, stdout); */ fwrite(buf, len, 1, stdout); if (ferror(stdout)) FATAL("write error on stdout"); } else { fp = redirect(ptoi(a[1]), a[2]); /* fputs(buf, fp); */ fwrite(buf, len, 1, fp); fflush(fp); if (ferror(fp)) FATAL("write error on %s", filename(fp)); } free(buf); return(True); } Cell *arith(Node **a, int n) /* a[0] + a[1], etc. also -a[0] */ { Awkfloat i, j = 0; double v; Cell *x, *y, *z; x = execute(a[0]); i = getfval(x); tempfree(x); if (n != UMINUS && n != UPLUS) { y = execute(a[1]); j = getfval(y); tempfree(y); } z = gettemp(); switch (n) { case ADD: i += j; break; case MINUS: i -= j; break; case MULT: i *= j; break; case DIVIDE: if (j == 0) FATAL("division by zero"); i /= j; break; case MOD: if (j == 0) FATAL("division by zero in mod"); modf(i/j, &v); i = i - j * v; break; case UMINUS: i = -i; break; case UPLUS: /* handled by getfval(), above */ break; case POWER: if (j >= 0 && modf(j, &v) == 0.0) /* pos integer exponent */ i = ipow(i, (int) j); else { errno = 0; i = errcheck(pow(i, j), "pow"); } break; default: /* can't happen */ FATAL("illegal arithmetic operator %d", n); } setfval(z, i); return(z); } double ipow(double x, int n) /* x**n. ought to be done by pow, but isn't always */ { double v; if (n <= 0) return 1; v = ipow(x, n/2); if (n % 2 == 0) return v * v; else return x * v * v; } Cell *incrdecr(Node **a, int n) /* a[0]++, etc. */ { Cell *x, *z; int k; Awkfloat xf; x = execute(a[0]); xf = getfval(x); k = (n == PREINCR || n == POSTINCR) ? 1 : -1; if (n == PREINCR || n == PREDECR) { setfval(x, xf + k); return(x); } z = gettemp(); setfval(z, xf); setfval(x, xf + k); tempfree(x); return(z); } Cell *assign(Node **a, int n) /* a[0] = a[1], a[0] += a[1], etc. */ { /* this is subtle; don't muck with it. */ Cell *x, *y; Awkfloat xf, yf; double v; y = execute(a[1]); x = execute(a[0]); if (n == ASSIGN) { /* ordinary assignment */ if (x == y && !(x->tval & (FLD|REC)) && x != nfloc) ; /* self-assignment: leave alone unless it's a field or NF */ else if ((y->tval & (STR|NUM)) == (STR|NUM)) { + yf = getfval(y); setsval(x, getsval(y)); - x->fval = getfval(y); + x->fval = yf; x->tval |= NUM; } else if (isstr(y)) setsval(x, getsval(y)); else if (isnum(y)) setfval(x, getfval(y)); else funnyvar(y, "read value of"); tempfree(y); return(x); } xf = getfval(x); yf = getfval(y); switch (n) { case ADDEQ: xf += yf; break; case SUBEQ: xf -= yf; break; case MULTEQ: xf *= yf; break; case DIVEQ: if (yf == 0) FATAL("division by zero in /="); xf /= yf; break; case MODEQ: if (yf == 0) FATAL("division by zero in %%="); modf(xf/yf, &v); xf = xf - yf * v; break; case POWEQ: if (yf >= 0 && modf(yf, &v) == 0.0) /* pos integer exponent */ xf = ipow(xf, (int) yf); else { errno = 0; xf = errcheck(pow(xf, yf), "pow"); } break; default: FATAL("illegal assignment operator %d", n); break; } tempfree(y); setfval(x, xf); return(x); } Cell *cat(Node **a, int q) /* a[0] cat a[1] */ { Cell *x, *y, *z; int n1, n2; char *s = NULL; int ssz = 0; x = execute(a[0]); n1 = strlen(getsval(x)); - adjbuf(&s, &ssz, n1, recsize, 0, "cat1"); + adjbuf(&s, &ssz, n1 + 1, recsize, 0, "cat1"); memcpy(s, x->sval, n1); + tempfree(x); + y = execute(a[1]); n2 = strlen(getsval(y)); adjbuf(&s, &ssz, n1 + n2 + 1, recsize, 0, "cat2"); memcpy(s + n1, y->sval, n2); s[n1 + n2] = '\0'; - tempfree(x); tempfree(y); z = gettemp(); z->sval = s; z->tval = STR; return(z); } Cell *pastat(Node **a, int n) /* a[0] { a[1] } */ { Cell *x; if (a[0] == NULL) x = execute(a[1]); else { x = execute(a[0]); if (istrue(x)) { tempfree(x); x = execute(a[1]); } } return x; } Cell *dopa2(Node **a, int n) /* a[0], a[1] { a[2] } */ { Cell *x; int pair; pair = ptoi(a[3]); if (pairstack[pair] == 0) { x = execute(a[0]); if (istrue(x)) pairstack[pair] = 1; tempfree(x); } if (pairstack[pair] == 1) { x = execute(a[1]); if (istrue(x)) pairstack[pair] = 0; tempfree(x); x = execute(a[2]); return(x); } return(False); } Cell *split(Node **a, int nnn) /* split(a[0], a[1], a[2]); a[3] is type */ { Cell *x = NULL, *y, *ap; const char *s, *origs, *t; const char *fs = NULL; char *origfs = NULL; int sep; char temp, num[50]; int n, tempstat, arg3type; + int j; double result; y = execute(a[0]); /* source string */ origs = s = strdup(getsval(y)); + tempfree(y); arg3type = ptoi(a[3]); - if (a[2] == NULL) /* fs string */ + if (a[2] == NULL) { /* BUG: CSV should override implicit fs but not explicit */ fs = getsval(fsloc); - else if (arg3type == STRING) { /* split(str,arr,"string") */ + } else if (arg3type == STRING) { /* split(str,arr,"string") */ x = execute(a[2]); fs = origfs = strdup(getsval(x)); tempfree(x); - } else if (arg3type == REGEXPR) + } else if (arg3type == REGEXPR) { fs = "(regexpr)"; /* split(str,arr,/regexpr/) */ - else + } else { FATAL("illegal type of split"); + } sep = *fs; ap = execute(a[1]); /* array name */ +/* BUG 7/26/22: this appears not to reset array: see C1/asplit */ freesymtab(ap); DPRINTF("split: s=|%s|, a=%s, sep=|%s|\n", s, NN(ap->nval), fs); ap->tval &= ~STR; ap->tval |= ARR; ap->sval = (char *) makesymtab(NSYMTAB); n = 0; if (arg3type == REGEXPR && strlen((char*)((fa*)a[2])->restr) == 0) { /* split(s, a, //); have to arrange that it looks like empty sep */ arg3type = 0; fs = ""; sep = 0; } if (*s != '\0' && (strlen(fs) > 1 || arg3type == REGEXPR)) { /* reg expr */ fa *pfa; if (arg3type == REGEXPR) { /* it's ready already */ pfa = (fa *) a[2]; } else { pfa = makedfa(fs, 1); } if (nematch(pfa,s)) { tempstat = pfa->initstat; pfa->initstat = 2; do { n++; snprintf(num, sizeof(num), "%d", n); temp = *patbeg; setptr(patbeg, '\0'); if (is_number(s, & result)) setsymtab(num, s, result, STR|NUM, (Array *) ap->sval); else setsymtab(num, s, 0.0, STR, (Array *) ap->sval); setptr(patbeg, temp); s = patbeg + patlen; if (*(patbeg+patlen-1) == '\0' || *s == '\0') { n++; snprintf(num, sizeof(num), "%d", n); setsymtab(num, "", 0.0, STR, (Array *) ap->sval); pfa->initstat = tempstat; goto spdone; } } while (nematch(pfa,s)); pfa->initstat = tempstat; /* bwk: has to be here to reset */ /* cf gsub and refldbld */ } n++; snprintf(num, sizeof(num), "%d", n); if (is_number(s, & result)) setsymtab(num, s, result, STR|NUM, (Array *) ap->sval); else setsymtab(num, s, 0.0, STR, (Array *) ap->sval); spdone: pfa = NULL; - } else if (sep == ' ') { + + } else if (a[2] == NULL && CSV) { /* CSV only if no explicit separator */ + char *newt = (char *) malloc(strlen(s)); /* for building new string; reuse for each field */ + for (;;) { + char *fr = newt; + n++; + if (*s == '"' ) { /* start of "..." */ + for (s++ ; *s != '\0'; ) { + if (*s == '"' && s[1] != '\0' && s[1] == '"') { + s += 2; /* doubled quote */ + *fr++ = '"'; + } else if (*s == '"' && (s[1] == '\0' || s[1] == ',')) { + s++; /* skip over closing quote */ + break; + } else { + *fr++ = *s++; + } + } + *fr++ = 0; + } else { /* unquoted field */ + while (*s != ',' && *s != '\0') + *fr++ = *s++; + *fr++ = 0; + } + snprintf(num, sizeof(num), "%d", n); + if (is_number(newt, &result)) + setsymtab(num, newt, result, STR|NUM, (Array *) ap->sval); + else + setsymtab(num, newt, 0.0, STR, (Array *) ap->sval); + if (*s++ == '\0') + break; + } + free(newt); + + } else if (!CSV && sep == ' ') { /* usual case: split on white space */ for (n = 0; ; ) { #define ISWS(c) ((c) == ' ' || (c) == '\t' || (c) == '\n') while (ISWS(*s)) s++; if (*s == '\0') break; n++; t = s; do s++; while (*s != '\0' && !ISWS(*s)); temp = *s; setptr(s, '\0'); snprintf(num, sizeof(num), "%d", n); if (is_number(t, & result)) setsymtab(num, t, result, STR|NUM, (Array *) ap->sval); else setsymtab(num, t, 0.0, STR, (Array *) ap->sval); setptr(s, temp); if (*s != '\0') s++; } + } else if (sep == 0) { /* new: split(s, a, "") => 1 char/elem */ - for (n = 0; *s != '\0'; s++) { - char buf[2]; + for (n = 0; *s != '\0'; s += u8_nextlen(s)) { + char buf[10]; n++; snprintf(num, sizeof(num), "%d", n); - buf[0] = *s; - buf[1] = '\0'; + + for (j = 0; j < u8_nextlen(s); j++) { + buf[j] = s[j]; + } + buf[j] = '\0'; + if (isdigit((uschar)buf[0])) setsymtab(num, buf, atof(buf), STR|NUM, (Array *) ap->sval); else setsymtab(num, buf, 0.0, STR, (Array *) ap->sval); } - } else if (*s != '\0') { + + } else if (*s != '\0') { /* some random single character */ for (;;) { n++; t = s; while (*s != sep && *s != '\n' && *s != '\0') s++; temp = *s; setptr(s, '\0'); snprintf(num, sizeof(num), "%d", n); if (is_number(t, & result)) setsymtab(num, t, result, STR|NUM, (Array *) ap->sval); else setsymtab(num, t, 0.0, STR, (Array *) ap->sval); setptr(s, temp); if (*s++ == '\0') break; } } tempfree(ap); - tempfree(y); xfree(origs); xfree(origfs); x = gettemp(); x->tval = NUM; x->fval = n; return(x); } Cell *condexpr(Node **a, int n) /* a[0] ? a[1] : a[2] */ { Cell *x; x = execute(a[0]); if (istrue(x)) { tempfree(x); x = execute(a[1]); } else { tempfree(x); x = execute(a[2]); } return(x); } Cell *ifstat(Node **a, int n) /* if (a[0]) a[1]; else a[2] */ { Cell *x; x = execute(a[0]); if (istrue(x)) { tempfree(x); x = execute(a[1]); } else if (a[2] != NULL) { tempfree(x); x = execute(a[2]); } return(x); } Cell *whilestat(Node **a, int n) /* while (a[0]) a[1] */ { Cell *x; for (;;) { x = execute(a[0]); if (!istrue(x)) return(x); tempfree(x); x = execute(a[1]); if (isbreak(x)) { x = True; return(x); } if (isnext(x) || isexit(x) || isret(x)) return(x); tempfree(x); } } Cell *dostat(Node **a, int n) /* do a[0]; while(a[1]) */ { Cell *x; for (;;) { x = execute(a[0]); if (isbreak(x)) return True; if (isnext(x) || isexit(x) || isret(x)) return(x); tempfree(x); x = execute(a[1]); if (!istrue(x)) return(x); tempfree(x); } } Cell *forstat(Node **a, int n) /* for (a[0]; a[1]; a[2]) a[3] */ { Cell *x; x = execute(a[0]); tempfree(x); for (;;) { if (a[1]!=NULL) { x = execute(a[1]); if (!istrue(x)) return(x); else tempfree(x); } x = execute(a[3]); if (isbreak(x)) /* turn off break */ return True; if (isnext(x) || isexit(x) || isret(x)) return(x); tempfree(x); x = execute(a[2]); tempfree(x); } } Cell *instat(Node **a, int n) /* for (a[0] in a[1]) a[2] */ { Cell *x, *vp, *arrayp, *cp, *ncp; Array *tp; int i; vp = execute(a[0]); arrayp = execute(a[1]); if (!isarr(arrayp)) { return True; } tp = (Array *) arrayp->sval; tempfree(arrayp); for (i = 0; i < tp->size; i++) { /* this routine knows too much */ for (cp = tp->tab[i]; cp != NULL; cp = ncp) { setsval(vp, cp->nval); ncp = cp->cnext; x = execute(a[2]); if (isbreak(x)) { tempfree(vp); return True; } if (isnext(x) || isexit(x) || isret(x)) { tempfree(vp); return(x); } tempfree(x); } } return True; } static char *nawk_convert(const char *s, int (*fun_c)(int), wint_t (*fun_wc)(wint_t)) { char *buf = NULL; char *pbuf = NULL; const char *ps = NULL; size_t n = 0; wchar_t wc; - size_t sz = MB_CUR_MAX; + const size_t sz = awk_mb_cur_max; + int unused; if (sz == 1) { buf = tostring(s); for (pbuf = buf; *pbuf; pbuf++) *pbuf = fun_c((uschar)*pbuf); return buf; } else { /* upper/lower character may be shorter/longer */ buf = tostringN(s, strlen(s) * sz + 1); (void) mbtowc(NULL, NULL, 0); /* reset internal state */ /* * Reset internal state here too. * Assign result to avoid a compiler warning. (Casting to void * doesn't work.) * Increment said variable to avoid a different warning. */ - int unused = wctomb(NULL, L'\0'); + unused = wctomb(NULL, L'\0'); unused++; ps = s; pbuf = buf; while (n = mbtowc(&wc, ps, sz), n > 0 && n != (size_t)-1 && n != (size_t)-2) { ps += n; n = wctomb(pbuf, fun_wc(wc)); if (n == (size_t)-1) FATAL("illegal wide character %s", s); pbuf += n; } *pbuf = '\0'; if (n) FATAL("illegal byte sequence %s", s); return buf; } } #ifdef __DJGPP__ static wint_t towupper(wint_t wc) { if (wc >= 0 && wc < 256) return toupper(wc & 0xFF); return wc; } static wint_t towlower(wint_t wc) { if (wc >= 0 && wc < 256) return tolower(wc & 0xFF); return wc; } #endif static char *nawk_toupper(const char *s) { return nawk_convert(s, toupper, towupper); } static char *nawk_tolower(const char *s) { return nawk_convert(s, tolower, towlower); } + + Cell *bltin(Node **a, int n) /* builtin functions. a[0] is type, a[1] is arg list */ { Cell *x, *y; Awkfloat u; int t, sz; Awkfloat tmp; char *buf, *fmt; Node *nextarg; FILE *fp; int status = 0; time_t tv; struct tm *tm; + int estatus = 0; t = ptoi(a[0]); x = execute(a[1]); nextarg = a[1]->nnext; switch (t) { case FLENGTH: if (isarr(x)) u = ((Array *) x->sval)->nelem; /* GROT. should be function*/ else - u = strlen(getsval(x)); + u = u8_strlen(getsval(x)); break; case FLOG: errno = 0; u = errcheck(log(getfval(x)), "log"); break; case FINT: modf(getfval(x), &u); break; case FEXP: errno = 0; u = errcheck(exp(getfval(x)), "exp"); break; case FSQRT: errno = 0; u = errcheck(sqrt(getfval(x)), "sqrt"); break; case FSIN: u = sin(getfval(x)); break; case FCOS: u = cos(getfval(x)); break; case FATAN: if (nextarg == NULL) { WARNING("atan2 requires two arguments; returning 1.0"); u = 1.0; } else { y = execute(a[1]->nnext); u = atan2(getfval(x), getfval(y)); tempfree(y); nextarg = nextarg->nnext; } break; case FCOMPL: u = ~((int)getfval(x)); break; case FAND: if (nextarg == 0) { WARNING("and requires two arguments; returning 0"); u = 0; break; } y = execute(a[1]->nnext); u = ((int)getfval(x)) & ((int)getfval(y)); tempfree(y); nextarg = nextarg->nnext; break; case FFOR: if (nextarg == 0) { WARNING("or requires two arguments; returning 0"); u = 0; break; } y = execute(a[1]->nnext); u = ((int)getfval(x)) | ((int)getfval(y)); tempfree(y); nextarg = nextarg->nnext; break; case FXOR: if (nextarg == 0) { WARNING("xor requires two arguments; returning 0"); u = 0; break; } y = execute(a[1]->nnext); u = ((int)getfval(x)) ^ ((int)getfval(y)); tempfree(y); nextarg = nextarg->nnext; break; case FLSHIFT: if (nextarg == 0) { WARNING("lshift requires two arguments; returning 0"); u = 0; break; } y = execute(a[1]->nnext); u = ((int)getfval(x)) << ((int)getfval(y)); tempfree(y); nextarg = nextarg->nnext; break; case FRSHIFT: if (nextarg == 0) { WARNING("rshift requires two arguments; returning 0"); u = 0; break; } y = execute(a[1]->nnext); u = ((int)getfval(x)) >> ((int)getfval(y)); tempfree(y); nextarg = nextarg->nnext; break; case FSYSTEM: fflush(stdout); /* in case something is buffered already */ - status = system(getsval(x)); - u = status; + estatus = status = system(getsval(x)); if (status != -1) { if (WIFEXITED(status)) { - u = WEXITSTATUS(status); + estatus = WEXITSTATUS(status); } else if (WIFSIGNALED(status)) { - u = WTERMSIG(status) + 256; + estatus = WTERMSIG(status) + 256; #ifdef WCOREDUMP if (WCOREDUMP(status)) - u += 256; + estatus += 256; #endif } else /* something else?!? */ - u = 0; + estatus = 0; } + /* else estatus was set to -1 */ + u = estatus; break; case FRAND: /* random() returns numbers in [0..2^31-1] * in order to get a number in [0, 1), divide it by 2^31 */ u = (Awkfloat) random() / (0x7fffffffL + 0x1UL); break; case FSRAND: if (isrec(x)) /* no argument provided */ u = time((time_t *)0); else u = getfval(x); tmp = u; srandom((unsigned long) u); u = srand_seed; srand_seed = tmp; break; case FTOUPPER: case FTOLOWER: if (t == FTOUPPER) buf = nawk_toupper(getsval(x)); else buf = nawk_tolower(getsval(x)); tempfree(x); x = gettemp(); setsval(x, buf); free(buf); return x; case FFLUSH: if (isrec(x) || strlen(getsval(x)) == 0) { flush_all(); /* fflush() or fflush("") -> all */ u = 0; } else if ((fp = openfile(FFLUSH, getsval(x), NULL)) == NULL) u = EOF; else u = fflush(fp); break; case FSYSTIME: u = time((time_t *) 0); break; case FSTRFTIME: /* strftime([format [,timestamp]]) */ if (nextarg) { y = execute(nextarg); nextarg = nextarg->nnext; tv = (time_t) getfval(y); tempfree(y); } else tv = time((time_t *) 0); tm = localtime(&tv); if (tm == NULL) FATAL("bad time %ld", (long)tv); if (isrec(x)) { /* format argument not provided, use default */ fmt = tostring("%a %b %d %H:%M:%S %Z %Y"); } else fmt = tostring(getsval(x)); sz = 32; buf = NULL; do { if ((buf = realloc(buf, (sz *= 2))) == NULL) FATAL("out of memory in strftime"); } while (strftime(buf, sz, fmt, tm) == 0 && fmt[0] != '\0'); y = gettemp(); setsval(y, buf); free(fmt); free(buf); return y; default: /* can't happen */ FATAL("illegal function type %d", t); break; } tempfree(x); x = gettemp(); setfval(x, u); if (nextarg != NULL) { WARNING("warning: function has too many arguments"); - for ( ; nextarg; nextarg = nextarg->nnext) - execute(nextarg); + for ( ; nextarg; nextarg = nextarg->nnext) { + y = execute(nextarg); + tempfree(y); + } } return(x); } Cell *printstat(Node **a, int n) /* print a[0] */ { Node *x; Cell *y; FILE *fp; if (a[1] == NULL) /* a[1] is redirection operator, a[2] is file */ fp = stdout; else fp = redirect(ptoi(a[1]), a[2]); for (x = a[0]; x != NULL; x = x->nnext) { y = execute(x); fputs(getpssval(y), fp); tempfree(y); if (x->nnext == NULL) fputs(getsval(orsloc), fp); else fputs(getsval(ofsloc), fp); } if (a[1] != NULL) fflush(fp); if (ferror(fp)) FATAL("write error on %s", filename(fp)); return(True); } Cell *nullproc(Node **a, int n) { return 0; } FILE *redirect(int a, Node *b) /* set up all i/o redirections */ { FILE *fp; Cell *x; char *fname; x = execute(b); fname = getsval(x); fp = openfile(a, fname, NULL); if (fp == NULL) FATAL("can't open file %s", fname); tempfree(x); return fp; } struct files { FILE *fp; const char *fname; int mode; /* '|', 'a', 'w' => LE/LT, GT */ } *files; size_t nfiles; static void stdinit(void) /* in case stdin, etc., are not constants */ { nfiles = FOPEN_MAX; files = (struct files *) calloc(nfiles, sizeof(*files)); if (files == NULL) FATAL("can't allocate file memory for %zu files", nfiles); files[0].fp = stdin; - files[0].fname = "/dev/stdin"; + files[0].fname = tostring("/dev/stdin"); files[0].mode = LT; files[1].fp = stdout; - files[1].fname = "/dev/stdout"; + files[1].fname = tostring("/dev/stdout"); files[1].mode = GT; files[2].fp = stderr; - files[2].fname = "/dev/stderr"; + files[2].fname = tostring("/dev/stderr"); files[2].mode = GT; } FILE *openfile(int a, const char *us, bool *pnewflag) { const char *s = us; size_t i; int m; FILE *fp = NULL; if (*s == '\0') FATAL("null file name in print or getline"); for (i = 0; i < nfiles; i++) if (files[i].fname && strcmp(s, files[i].fname) == 0 && (a == files[i].mode || (a==APPEND && files[i].mode==GT) || a == FFLUSH)) { if (pnewflag) *pnewflag = false; return files[i].fp; } if (a == FFLUSH) /* didn't find it, so don't create it! */ return NULL; for (i = 0; i < nfiles; i++) if (files[i].fp == NULL) break; if (i >= nfiles) { struct files *nf; size_t nnf = nfiles + FOPEN_MAX; nf = (struct files *) realloc(files, nnf * sizeof(*nf)); if (nf == NULL) FATAL("cannot grow files for %s and %zu files", s, nnf); memset(&nf[nfiles], 0, FOPEN_MAX * sizeof(*nf)); nfiles = nnf; files = nf; } fflush(stdout); /* force a semblance of order */ m = a; if (a == GT) { fp = fopen(s, "w"); } else if (a == APPEND) { fp = fopen(s, "a"); m = GT; /* so can mix > and >> */ } else if (a == '|') { /* output pipe */ fp = popen(s, "w"); } else if (a == LE) { /* input pipe */ fp = popen(s, "r"); } else if (a == LT) { /* getline sval, files[i].fname) != 0) continue; - if (ferror(files[i].fp)) - FATAL("i/o error occurred on %s", files[i].fname); + if (files[i].mode == GT || files[i].mode == '|') + fflush(files[i].fp); + if (ferror(files[i].fp)) { + if ((files[i].mode == GT && files[i].fp != stderr) + || files[i].mode == '|') + FATAL("write error on %s", files[i].fname); + else + WARNING("i/o error occurred on %s", files[i].fname); + } if (files[i].fp == stdin || files[i].fp == stdout || files[i].fp == stderr) stat = freopen("/dev/null", "r+", files[i].fp) == NULL; else if (files[i].mode == '|' || files[i].mode == LE) stat = pclose(files[i].fp) == -1; else stat = fclose(files[i].fp) == EOF; if (stat) - FATAL("i/o error occurred closing %s", files[i].fname); - if (i > 2) /* don't do /dev/std... */ - xfree(files[i].fname); + WARNING("i/o error occurred closing %s", files[i].fname); + xfree(files[i].fname); files[i].fname = NULL; /* watch out for ref thru this */ files[i].fp = NULL; break; } tempfree(x); x = gettemp(); setfval(x, (Awkfloat) (stat ? -1 : 0)); return(x); - } +} void closeall(void) { size_t i; bool stat = false; for (i = 0; i < nfiles; i++) { if (! files[i].fp) continue; - if (ferror(files[i].fp)) - FATAL( "i/o error occurred on %s", files[i].fname ); - if (files[i].fp == stdin) + if (files[i].mode == GT || files[i].mode == '|') + fflush(files[i].fp); + if (ferror(files[i].fp)) { + if ((files[i].mode == GT && files[i].fp != stderr) + || files[i].mode == '|') + FATAL("write error on %s", files[i].fname); + else + WARNING("i/o error occurred on %s", files[i].fname); + } + if (files[i].fp == stdin || files[i].fp == stdout || + files[i].fp == stderr) continue; if (files[i].mode == '|' || files[i].mode == LE) stat = pclose(files[i].fp) == -1; - else if (files[i].fp == stdout || files[i].fp == stderr) - stat = fflush(files[i].fp) == EOF; else stat = fclose(files[i].fp) == EOF; if (stat) - FATAL( "i/o error occurred while closing %s", files[i].fname ); + WARNING("i/o error occurred while closing %s", files[i].fname); } } static void flush_all(void) { size_t i; for (i = 0; i < nfiles; i++) if (files[i].fp) fflush(files[i].fp); } void backsub(char **pb_ptr, const char **sptr_ptr); -Cell *sub(Node **a, int nnn) /* substitute command */ +Cell *dosub(Node **a, int subop) /* sub and gsub */ { - const char *sptr, *q; - Cell *x, *y, *result; - char *t, *buf, *pb; fa *pfa; + int tempstat; + char *repl; + Cell *x; + + char *buf = NULL; + char *pb = NULL; int bufsz = recsize; - if ((buf = (char *) malloc(bufsz)) == NULL) - FATAL("out of memory in sub"); - x = execute(a[3]); /* target string */ - t = getsval(x); - if (a[0] == NULL) /* 0 => a[1] is already-compiled regexpr */ - pfa = (fa *) a[1]; /* regular expression */ - else { - y = execute(a[1]); - pfa = makedfa(getsval(y), 1); - tempfree(y); + const char *r, *s; + const char *start; + const char *noempty = NULL; /* empty match disallowed here */ + size_t m = 0; /* match count */ + size_t whichm; /* which match to select, 0 = global */ + int mtype; /* match type */ + + if (a[0] == NULL) { /* 0 => a[1] is already-compiled regexpr */ + pfa = (fa *) a[1]; + } else { + x = execute(a[1]); + pfa = makedfa(getsval(x), 1); + tempfree(x); } - y = execute(a[2]); /* replacement string */ - result = False; - if (pmatch(pfa, t)) { - sptr = t; - adjbuf(&buf, &bufsz, 1+patbeg-sptr, recsize, 0, "sub"); - pb = buf; - while (sptr < patbeg) - *pb++ = *sptr++; - sptr = getsval(y); - while (*sptr != '\0') { - adjbuf(&buf, &bufsz, 5+pb-buf, recsize, &pb, "sub"); - if (*sptr == '\\') { - backsub(&pb, &sptr); - } else if (*sptr == '&') { - sptr++; - adjbuf(&buf, &bufsz, 1+patlen+pb-buf, recsize, &pb, "sub"); - for (q = patbeg; q < patbeg+patlen; ) - *pb++ = *q++; - } else - *pb++ = *sptr++; + + x = execute(a[2]); /* replacement string */ + repl = tostring(getsval(x)); + tempfree(x); + + switch (subop) { + case SUB: + whichm = 1; + x = execute(a[3]); /* source string */ + break; + case GSUB: + whichm = 0; + x = execute(a[3]); /* source string */ + break; + default: + FATAL("dosub: unrecognized subop: %d", subop); + } + + start = getsval(x); + while (pmatch(pfa, start)) { + if (buf == NULL) { + if ((pb = buf = (char *) malloc(bufsz)) == NULL) + FATAL("out of memory in dosub"); + tempstat = pfa->initstat; + pfa->initstat = 2; } - *pb = '\0'; - if (pb > buf + bufsz) - FATAL("sub result1 %.30s too big; can't happen", buf); - sptr = patbeg + patlen; - if ((patlen == 0 && *patbeg) || (patlen && *(sptr-1))) { - adjbuf(&buf, &bufsz, 1+strlen(sptr)+pb-buf, 0, &pb, "sub"); - while ((*pb++ = *sptr++) != '\0') - continue; + + /* match types */ + #define MT_IGNORE 0 /* unselected or invalid */ + #define MT_INSERT 1 /* selected, empty */ + #define MT_REPLACE 2 /* selected, not empty */ + + /* an empty match just after replacement is invalid */ + + if (patbeg == noempty && patlen == 0) { + mtype = MT_IGNORE; /* invalid, not counted */ + } else if (whichm == ++m || whichm == 0) { + mtype = patlen ? MT_REPLACE : MT_INSERT; + } else { + mtype = MT_IGNORE; /* unselected, but counted */ } - if (pb > buf + bufsz) - FATAL("sub result2 %.30s too big; can't happen", buf); - setsval(x, buf); /* BUG: should be able to avoid copy */ - result = True; - } - tempfree(x); - tempfree(y); - free(buf); - return result; -} -Cell *gsub(Node **a, int nnn) /* global substitute */ -{ - Cell *x, *y; - char *rptr, *pb; - const char *q, *t, *sptr; - char *buf; - fa *pfa; - int mflag, tempstat, num; - int bufsz = recsize; + /* leading text: */ + if (patbeg > start) { + adjbuf(&buf, &bufsz, (pb - buf) + (patbeg - start), + recsize, &pb, "dosub"); + s = start; + while (s < patbeg) + *pb++ = *s++; + } - if ((buf = (char *) malloc(bufsz)) == NULL) - FATAL("out of memory in gsub"); - mflag = 0; /* if mflag == 0, can replace empty string */ - num = 0; - x = execute(a[3]); /* target string */ - t = getsval(x); - if (a[0] == NULL) /* 0 => a[1] is already-compiled regexpr */ - pfa = (fa *) a[1]; /* regular expression */ - else { - y = execute(a[1]); - pfa = makedfa(getsval(y), 1); - tempfree(y); - } - y = execute(a[2]); /* replacement string */ - if (pmatch(pfa, t)) { - tempstat = pfa->initstat; - pfa->initstat = 2; - pb = buf; - rptr = getsval(y); - do { - if (patlen == 0 && *patbeg != '\0') { /* matched empty string */ - if (mflag == 0) { /* can replace empty */ - num++; - sptr = rptr; - while (*sptr != '\0') { - adjbuf(&buf, &bufsz, 5+pb-buf, recsize, &pb, "gsub"); - if (*sptr == '\\') { - backsub(&pb, &sptr); - } else if (*sptr == '&') { - sptr++; - adjbuf(&buf, &bufsz, 1+patlen+pb-buf, recsize, &pb, "gsub"); - for (q = patbeg; q < patbeg+patlen; ) - *pb++ = *q++; - } else - *pb++ = *sptr++; - } - } - if (*t == '\0') /* at end */ - goto done; - adjbuf(&buf, &bufsz, 2+pb-buf, recsize, &pb, "gsub"); - *pb++ = *t++; - if (pb > buf + bufsz) /* BUG: not sure of this test */ - FATAL("gsub result0 %.30s too big; can't happen", buf); - mflag = 0; + if (mtype == MT_IGNORE) + goto matching_text; /* skip replacement text */ + + r = repl; + while (*r != 0) { + adjbuf(&buf, &bufsz, 5+pb-buf, recsize, &pb, "dosub"); + if (*r == '\\') { + backsub(&pb, &r); + } else if (*r == '&') { + r++; + adjbuf(&buf, &bufsz, 1+patlen+pb-buf, recsize, + &pb, "dosub"); + for (s = patbeg; s < patbeg+patlen; ) + *pb++ = *s++; + } else { + *pb++ = *r++; } - else { /* matched nonempty string */ - num++; - sptr = t; - adjbuf(&buf, &bufsz, 1+(patbeg-sptr)+pb-buf, recsize, &pb, "gsub"); - while (sptr < patbeg) - *pb++ = *sptr++; - sptr = rptr; - while (*sptr != '\0') { - adjbuf(&buf, &bufsz, 5+pb-buf, recsize, &pb, "gsub"); - if (*sptr == '\\') { - backsub(&pb, &sptr); - } else if (*sptr == '&') { - sptr++; - adjbuf(&buf, &bufsz, 1+patlen+pb-buf, recsize, &pb, "gsub"); - for (q = patbeg; q < patbeg+patlen; ) - *pb++ = *q++; - } else - *pb++ = *sptr++; - } - t = patbeg + patlen; - if (patlen == 0 || *t == '\0' || *(t-1) == '\0') - goto done; - if (pb > buf + bufsz) - FATAL("gsub result1 %.30s too big; can't happen", buf); - mflag = 1; - } - } while (pmatch(pfa,t)); - sptr = t; - adjbuf(&buf, &bufsz, 1+strlen(sptr)+pb-buf, 0, &pb, "gsub"); - while ((*pb++ = *sptr++) != '\0') - continue; - done: if (pb < buf + bufsz) - *pb = '\0'; - else if (*(pb-1) != '\0') - FATAL("gsub result2 %.30s truncated; can't happen", buf); - setsval(x, buf); /* BUG: should be able to avoid copy + free */ + } + +matching_text: + if (mtype == MT_REPLACE || *patbeg == '\0') + goto next_search; /* skip matching text */ + + if (patlen == 0) + patlen = u8_nextlen(patbeg); + adjbuf(&buf, &bufsz, (pb-buf) + patlen, recsize, &pb, "dosub"); + s = patbeg; + while (s < patbeg + patlen) + *pb++ = *s++; + +next_search: + start = patbeg + patlen; + if (m == whichm || *patbeg == '\0') + break; + if (mtype == MT_REPLACE) + noempty = start; + + #undef MT_IGNORE + #undef MT_INSERT + #undef MT_REPLACE + } + + xfree(repl); + + if (buf != NULL) { pfa->initstat = tempstat; + + /* trailing text */ + adjbuf(&buf, &bufsz, 1+strlen(start)+pb-buf, 0, &pb, "dosub"); + while ((*pb++ = *start++) != '\0') + ; + + setsval(x, buf); + free(buf); } + tempfree(x); - tempfree(y); x = gettemp(); x->tval = NUM; - x->fval = num; - free(buf); - return(x); + x->fval = m; + return x; } Cell *gensub(Node **a, int nnn) /* global selective substitute */ /* XXX incomplete - doesn't support backreferences \0 ... \9 */ { Cell *x, *y, *res, *h; char *rptr; const char *sptr; char *buf, *pb; const char *t, *q; fa *pfa; int mflag, tempstat, num, whichm; int bufsz = recsize; if ((buf = malloc(bufsz)) == NULL) FATAL("out of memory in gensub"); mflag = 0; /* if mflag == 0, can replace empty string */ num = 0; x = execute(a[4]); /* source string */ t = getsval(x); res = copycell(x); /* target string - initially copy of source */ res->csub = CTEMP; /* result values are temporary */ if (a[0] == 0) /* 0 => a[1] is already-compiled regexpr */ pfa = (fa *) a[1]; /* regular expression */ else { y = execute(a[1]); pfa = makedfa(getsval(y), 1); tempfree(y); } y = execute(a[2]); /* replacement string */ h = execute(a[3]); /* which matches should be replaced */ sptr = getsval(h); if (sptr[0] == 'g' || sptr[0] == 'G') whichm = -1; else { /* * The specified number is index of replacement, starting * from 1. GNU awk treats index lower than 0 same as * 1, we do same for compatibility. */ whichm = (int) getfval(h) - 1; if (whichm < 0) whichm = 0; } tempfree(h); if (pmatch(pfa, t)) { char *sl; tempstat = pfa->initstat; pfa->initstat = 2; pb = buf; rptr = getsval(y); /* * XXX if there are any backreferences in subst string, * complain now. */ for (sl = rptr; (sl = strchr(sl, '\\')) && sl[1]; sl++) { if (strchr("0123456789", sl[1])) { FATAL("gensub doesn't support backreferences (subst \"%s\")", rptr); } } do { if (whichm >= 0 && whichm != num) { num++; adjbuf(&buf, &bufsz, (pb - buf) + (patbeg - t) + patlen, recsize, &pb, "gensub"); /* copy the part of string up to and including * match to output buffer */ while (t < patbeg + patlen) *pb++ = *t++; continue; } if (patlen == 0 && *patbeg != 0) { /* matched empty string */ if (mflag == 0) { /* can replace empty */ num++; sptr = rptr; while (*sptr != 0) { adjbuf(&buf, &bufsz, 5+pb-buf, recsize, &pb, "gensub"); if (*sptr == '\\') { backsub(&pb, &sptr); } else if (*sptr == '&') { sptr++; adjbuf(&buf, &bufsz, 1+patlen+pb-buf, recsize, &pb, "gensub"); for (q = patbeg; q < patbeg+patlen; ) *pb++ = *q++; } else *pb++ = *sptr++; } } if (*t == 0) /* at end */ goto done; adjbuf(&buf, &bufsz, 2+pb-buf, recsize, &pb, "gensub"); *pb++ = *t++; if (pb > buf + bufsz) /* BUG: not sure of this test */ FATAL("gensub result0 %.30s too big; can't happen", buf); mflag = 0; } else { /* matched nonempty string */ num++; sptr = t; adjbuf(&buf, &bufsz, 1+(patbeg-sptr)+pb-buf, recsize, &pb, "gensub"); while (sptr < patbeg) *pb++ = *sptr++; sptr = rptr; while (*sptr != 0) { adjbuf(&buf, &bufsz, 5+pb-buf, recsize, &pb, "gensub"); if (*sptr == '\\') { backsub(&pb, &sptr); } else if (*sptr == '&') { sptr++; adjbuf(&buf, &bufsz, 1+patlen+pb-buf, recsize, &pb, "gensub"); for (q = patbeg; q < patbeg+patlen; ) *pb++ = *q++; } else *pb++ = *sptr++; } t = patbeg + patlen; if (patlen == 0 || *t == 0 || *(t-1) == 0) goto done; if (pb > buf + bufsz) FATAL("gensub result1 %.30s too big; can't happen", buf); mflag = 1; } } while (pmatch(pfa,t)); sptr = t; adjbuf(&buf, &bufsz, 1+strlen(sptr)+pb-buf, 0, &pb, "gensub"); while ((*pb++ = *sptr++) != 0) ; done: if (pb > buf + bufsz) FATAL("gensub result2 %.30s too big; can't happen", buf); *pb = '\0'; setsval(res, buf); pfa->initstat = tempstat; } tempfree(x); tempfree(y); free(buf); return(res); } void backsub(char **pb_ptr, const char **sptr_ptr) /* handle \\& variations */ { /* sptr[0] == '\\' */ char *pb = *pb_ptr; const char *sptr = *sptr_ptr; static bool first = true; static bool do_posix = false; if (first) { first = false; do_posix = (getenv("POSIXLY_CORRECT") != NULL); } if (sptr[1] == '\\') { if (sptr[2] == '\\' && sptr[3] == '&') { /* \\\& -> \& */ *pb++ = '\\'; *pb++ = '&'; sptr += 4; } else if (sptr[2] == '&') { /* \\& -> \ + matched */ *pb++ = '\\'; sptr += 2; } else if (do_posix) { /* \\x -> \x */ sptr++; *pb++ = *sptr++; } else { /* \\x -> \\x */ *pb++ = *sptr++; *pb++ = *sptr++; } } else if (sptr[1] == '&') { /* literal & */ sptr++; *pb++ = *sptr++; } else /* literal \ */ *pb++ = *sptr++; *pb_ptr = pb; *sptr_ptr = sptr; } + +static char *wide_char_to_byte_str(int rune, size_t *outlen) +{ + static char buf[5]; + int len; + + if (rune < 0 || rune > 0x10FFFF) + return NULL; + + memset(buf, 0, sizeof(buf)); + + len = 0; + if (rune <= 0x0000007F) { + buf[len++] = rune; + } else if (rune <= 0x000007FF) { + // 110xxxxx 10xxxxxx + buf[len++] = 0xC0 | (rune >> 6); + buf[len++] = 0x80 | (rune & 0x3F); + } else if (rune <= 0x0000FFFF) { + // 1110xxxx 10xxxxxx 10xxxxxx + buf[len++] = 0xE0 | (rune >> 12); + buf[len++] = 0x80 | ((rune >> 6) & 0x3F); + buf[len++] = 0x80 | (rune & 0x3F); + + } else { + // 0x00010000 - 0x10FFFF + // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + buf[len++] = 0xF0 | (rune >> 18); + buf[len++] = 0x80 | ((rune >> 12) & 0x3F); + buf[len++] = 0x80 | ((rune >> 6) & 0x3F); + buf[len++] = 0x80 | (rune & 0x3F); + } + + *outlen = len; + buf[len++] = '\0'; + + return buf; +} diff --git a/contrib/one-true-awk/testdir/Compare.tt b/contrib/one-true-awk/testdir/Compare.tt index ca828d258658..4b297d731c94 100755 --- a/contrib/one-true-awk/testdir/Compare.tt +++ b/contrib/one-true-awk/testdir/Compare.tt @@ -1,49 +1,49 @@ #!/bin/sh oldawk=${oldawk-awk} awk=${awk-../a.out} echo compiling time.c -gcc time.c -o time +cc time.c -o time time=./time echo time command = $time #case `uname` in #SunOS) # time=/usr/bin/time ;; #Linux) # time=/usr/bin/time ;; #*) # time=time ;; #esac echo oldawk = $oldawk, awk = $awk, time command = $time # an arbitrary collection of input data cat td.1 td.1 >foo.td sed 's/^........................//' td.1 >>foo.td pr -m td.1 td.1 td.1 >>foo.td pr -2 td.1 >>foo.td cat bib >>foo.td wc foo.td td=foo.td >footot for i in $* do echo $i "($oldawk vs $awk)": # ind <$i $time $oldawk -f $i $td >foo2 2>foo2t cat foo2t $time $awk -f $i $td >foo1 2>foo1t cat foo1t cmp foo1 foo2 echo $i: >>footot cat foo1t foo2t >>footot done ctimes footot diff --git a/contrib/one-true-awk/testdir/REGRESS b/contrib/one-true-awk/testdir/REGRESS index 5c3667f5eede..b54ce3f68ea0 100755 --- a/contrib/one-true-awk/testdir/REGRESS +++ b/contrib/one-true-awk/testdir/REGRESS @@ -1,21 +1,21 @@ #!/bin/sh uname -a -gcc echo.c -o echo && echo echo compiled +cc echo.c -o echo && echo echo compiled oldawk=${oldawk-awk} awk=${awk-../a.out} echo oldawk=$oldawk, awk=$awk oldawk=$oldawk awk=$awk Compare.t t.* echo `ls t.* | wc -l` tests; echo oldawk=$oldawk awk=$awk Compare.p p.? p.??* echo `ls p.* | wc -l` tests; echo oldawk=$oldawk awk=$awk Compare.T1 echo `grep '\$awk' T.* | wc -l` tests; echo oldawk=$oldawk awk=$awk Compare.tt tt.* echo `ls tt.* | wc -l` tests; echo diff --git a/contrib/one-true-awk/testdir/T.argv b/contrib/one-true-awk/testdir/T.argv index b9a67ef39099..55e2754cd8b5 100755 --- a/contrib/one-true-awk/testdir/T.argv +++ b/contrib/one-true-awk/testdir/T.argv @@ -1,144 +1,150 @@ echo T.argv: misc tests of argc and argv awk=${awk-../a.out} echo >foo1 echo >foo2 $awk ' BEGIN { for (i = 1; i < ARGC-1; i++) printf "%s ", ARGV[i] if (ARGC > 1) printf "%s", ARGV[i] printf "\n" exit }' * >foo1 echo * >foo2 diff foo1 foo2 || echo 'BAD: T.argv (echo1 *)' $awk ' BEGIN { for (i = 1; i < ARGC; i++) { printf "%s", ARGV[i] if (i < ARGC-1) printf " " } printf "\n" exit }' * >foo1 echo * >foo2 diff foo1 foo2 || echo 'BAD: T.argv (echo2 *)' $awk ' BEGIN { print ARGC ARGV[ARGC-1] = "" for (i=0; i < ARGC; i++) print ARGV[i] exit } ' a bc def gh >foo1 echo "5 $awk a bc def " >foo2 diff foo1 foo2 || echo 'BAD: T.argv (argc *)' echo '1 2 3' >foo0 echo 'foo1 foo2 foo3' >foo1 $awk '{print L $0}' L=foo foo2 diff foo1 foo2 || echo 'BAD: T.argv (L=foo foo0 echo 'foo1 foo2 foo3' >foo1 $awk '{print L $0}' L=foo foo0 >foo2 diff foo1 foo2 || echo 'BAD: T.argv (L=foo foo1)' echo '1 2 3' >foo0 echo 'foo1 foo2 foo3' >foo1 cat foo0 | $awk '{print L $0}' L=foo - >foo2 diff foo1 foo2 || echo 'BAD: T.argv (L=foo -)' echo '1 2 3' >foo0 echo 'foo1 foo2 foo3 glop1 glop2 glop3' >foo1 $awk '{print L $0}' L=foo foo0 L=glop foo0 >foo2 diff foo1 foo2 || echo 'BAD: T.argv (L=foo L=glop)' echo '1 2 3' >foo0 echo '111 112 113 221 222 223' >foo1 $awk '{print L $0}' L=11 foo0 L=22 foo0 >foo2 diff foo1 foo2 || echo 'BAD: T.argv (L=11 L=22)' +echo >foo0 +echo 'name=value +name=value' >foo1 +$awk 'BEGIN { print ARGV[1] } { print ARGV[1] }' name=value foo0 >foo2 +diff foo1 foo2 || echo 'BAD: T.argv assignment operand modified' + echo 3.345 >foo1 $awk 'BEGIN { print ARGV[1] + ARGV[2]}' 1 2.345 >foo2 diff foo1 foo2 || echo 'BAD: T.argv (ARGV[1] + ARGV[2])' echo 3.345 >foo1 x1=1 x2=2.345 $awk 'BEGIN { print ENVIRON["x1"] + ENVIRON["x2"]}' 1 2.345 >foo2 diff foo1 foo2 || echo 'BAD: T.argv (ENVIRON[x1] + ENVIRON[x2])' echo 'foo1' >foo1 echo 'foo2' >foo2 echo 'foo3' >foo3 $awk 'BEGIN { ARGV[2] = "" } { print }' foo1 foo2 foo3 >foo4 echo 'foo1 foo3' >foo5 diff foo4 foo5 || echo 'BAD: T.argv zap ARGV[2]' echo hi > foo1 ; mv foo1 foo2 $awk 'BEGIN { ARGV[1] = "foo2" ; print FILENAME } { print FILENAME }' foo1 >foo3 echo ' foo2' >foo4 diff foo3 foo4 || echo 'BAD: T.argv startup FILENAME' # assumes that startup FILENAME is "" # test data balanced on pinhead... echo 'ARGV[3] is /dev/null ARGV[0] is ../a.out ARGV[1] is /dev/null' >foo1 $awk 'BEGIN { # this is a variant of arnolds original example ARGV[1] = "/dev/null" ARGV[2] = "glotch" # file open must skipped deleted argv ARGV[3] = "/dev/null" ARGC = 4 delete ARGV[2] } # note that input is read here END { for (i in ARGV) printf("ARGV[%d] is %s\n", i, ARGV[i]) }' >foo2 diff foo1 foo2 || echo 'BAD: T.argv delete ARGV[2]' diff --git a/contrib/one-true-awk/testdir/T.csv b/contrib/one-true-awk/testdir/T.csv new file mode 100755 index 000000000000..e0f3d708edaf --- /dev/null +++ b/contrib/one-true-awk/testdir/T.csv @@ -0,0 +1,80 @@ +#!/bin/sh + +echo T.csv: tests of csv field splitting, no embedded newlines + +awk=${awk-../a.out} + +$awk ' +BEGIN { + FS = "\t" + awk = "../a.out --csv" +} +NF == 0 || $1 ~ /^#/ { + next +} +$1 ~ /try/ { # new test + nt++ + sub(/try /, "") + prog = $0 + printf("%3d %s\n", nt, prog) + prog = sprintf("%s '"'"'%s'"'"'", awk, prog) + # print "prog is", prog + nt2 = 0 + while (getline > 0) { + if (NF == 0) # blank line terminates a sequence + break + input = $1 + for (i = 2; i < NF; i++) # input data + input = input "\t" $i + test = sprintf("./echo '"'"'%s'"'"' | %s >foo1; ", + input, prog) + if ($NF == "\"\"") + output = ">foo2;" + else + output = sprintf("./echo '"'"'%s'"'"' >foo2; ", $NF) + gsub(/\\t/, "\t", output) + gsub(/\\n/, "\n", output) + run = sprintf("cmp foo1 foo2 || echo test %d.%d failed", + nt, ++nt2) + # print "input is", input + # print "test is", test + # print "output is", output + # print "run is", run + system(test output run) + } + tt += nt2 +} +END { print tt, "tests" } +' <<\!!!! +# General format: +# try program as rest of line +# $1 $2 $3 output1 (\t for tab, \n for newline, +# $1 $2 $3 output2 ("" for null) +# ... terminated by blank line + + +try { for (i=1; i<=NF; i++) printf("[%s]", $i); printf("\n") } +a [a] + a [ a] +,a [][a] + , a [ ][ a] +a,b [a][b] +a,b,c [a][b][c] +"" [] +"abc" [abc] +"a""b" [a"b] +"a","b" [a][b] +a""b [a""b] +"a,b" [a,b] +"""" ["] +"""""" [""] +"""x""" ["x"] +""",""" [","] +,,"" [][][] +a""b [a""b] +a''b [a''b] +,, [][][] +a, [a][] +"", [][] +, [][] +!!!! diff --git a/contrib/one-true-awk/testdir/T.flags b/contrib/one-true-awk/testdir/T.flags index 33d7c8db7fe6..17ce56133745 100755 --- a/contrib/one-true-awk/testdir/T.flags +++ b/contrib/one-true-awk/testdir/T.flags @@ -1,24 +1,25 @@ echo T.flags: test some commandline flags awk=${awk-../a.out} $awk >foo 2>&1 grep '[Uu]sage' foo >/dev/null || echo 'T.flags: bad usage' $awk -f >foo 2>&1 grep 'no program' foo >/dev/null || echo 'T.flags: bad no program' $awk -f glop/glop >foo 2>&1 grep 'can.t open' foo >/dev/null || echo 'T.flags: bad can.t open program' $awk -fglop/glop >foo 2>&1 grep 'can.t open' foo >/dev/null || echo 'T.flags: bad can.t open program 2' $awk -zz 'BEGIN{}' >foo 2>&1 grep 'unknown option' foo >/dev/null || echo 'T.flags: bad unknown option' $awk -F >foo 2>&1 grep 'no field separator' foo >/dev/null || echo 'T.flags: bad missing field separator' -$awk -F '' >foo 2>&1 -grep 'field separator FS is empty' foo >/dev/null || echo 'T.flags: bad empty field separator' +### Awk is now like gawk and splits into separate characters if FS = "" +# $awk -F '' >foo 2>&1 +# grep 'field separator FS is empty' foo >/dev/null || echo 'T.flags: bad empty field separator' diff --git a/contrib/one-true-awk/testdir/T.misc b/contrib/one-true-awk/testdir/T.misc index ad34ab8fdd8b..b8ed3c1c45f9 100755 --- a/contrib/one-true-awk/testdir/T.misc +++ b/contrib/one-true-awk/testdir/T.misc @@ -1,506 +1,526 @@ #!/bin/sh echo T.misc: miscellaneous buglets now watched for awk=${awk-../a.out} rm -f core echo 'The big brown over the lazy doe The big brown over the lazy dog x The big brown over the lazy dog' >foo echo 'failed succeeded failed succeeded' >foo1 $awk '{ if (match($0, /^The big brown over the lazy dog/) == 0) { printf("failed\n") } else { printf("succeeded\n") } } ' foo >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc ghosh RE bug' echo '123 1234567890 12345678901' >foo echo '12345678901' >foo1 $awk 'length($0) > 10' foo >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc last number bug' # check some \ sequences in strings (ascii) echo HIJKL >foo1 echo foo | $awk '{ print "H\x49\x4a\x4BL" }' >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc hex string cvt' echo 012x45 >foo1 $awk 'BEGIN { print "0\061\62x\0645" }' >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc oct string cvt' # $i++ means ($i)++ echo 3 5 | $awk '{ i = 1; print $i++ ; print $1, i }' >foo1 echo '3 4 1' >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc bad field increment' # makes sure that fields are recomputed even if self-assignment # take into account that subtracting from NF now rebuilds the record echo 'a b c s p q r x y z' >foo echo 'a s p x' >foo1 $awk '{ NF -= 2; $1 = $1; print }' foo2 diff foo1 foo2 || echo 1>&2 "BAD: T.misc bad field self-assignment" echo '1 1' >foo1 $awk 'BEGIN {x = 1; print x; x = x; print x}' >foo2 diff foo1 foo2 || echo 1>&2 "BAD: T.misc bad self-assignment" echo 573109312 | $awk '{print $1*4}' >foo1 echo 2292437248 >foo2 diff foo1 foo2 || echo 1>&2 "BAD: T.misc bad overflow" # note that there are 8-bit characters in the echo # some shells will probably screw this up. echo '# code € 1 code € 2' | $awk '/^#/' >foo1 echo '#' >foo2 diff foo1 foo2 || echo 1>&2 "BAD: T.misc bad match of 8-bit char" echo hello | $awk 'BEGIN { FILENAME = "/etc/passwd" } { print $0 }' >/dev/null if test -r core; then echo 1>&2 "BAD: T.misc /etc/passwd dropped core"; fi echo hello | $awk ' function foo(foo) { foo = 1 foo() } { foo(bar) } ' >/dev/null 2>&1 if test -r core; then echo 1>&2 "BAD: T.misc function foo(foo) dropped core" rm -f core fi echo '2 10' | $awk '{ x[NR] = $0 } # test whether $0 is NUM as well as STR END { if (x[1] > x[2]) print "BAD: T.misc: $0 is not NUM" }' $awk 'BEGIN { npad = substr("alexander" " ",1,15) print npad }' >foo grep '\\' foo && echo 1>&2 "BAD: T.misc alexander fails" # This should give an error about function arguments $awk ' function foo(x) { print "x is" x } BEGIN { foo(foo) } ' 2>foo grep "can't use function foo" foo >/dev/null || echo 1>&2 "BAD: T.misc fcn args" # gawk defref test; should give error about undefined function $awk 'BEGIN { foo() }' 2>foo grep "calling undefined function foo" foo >/dev/null || echo 1>&2 "BAD: T.misc undefined function" # gawk arrayparm test; should give error about function $awk ' BEGIN { foo[1]=1; foo[2]=2; bug1(foo); } function bug1(i) { for (i in foo) { bug2(i); delete foo[i]; print i,1,bot[1]; } } function bug2(arg) { bot[arg]=arg; } ' 2>foo grep "can.t assign to foo" foo >/dev/null || echo 1>&2 "BAD: T.misc foo bug" # This should be a syntax error $awk ' !x = y ' 2>foo grep "syntax error" foo >/dev/null || echo 1>&2 "BAD: T.misc syntax error !x=y fails" # This should print bbb $awk ' BEGIN { up[1] = "a" for (i in up) gsub("a", "A", x) print x x "bbb" exit } ' >foo grep bbb foo >/dev/null || echo 1>&2 "BAD: T.misc gsub failed" echo yes | $awk ' BEGIN { printf "push return" >"/dev/null" getline ans <"/dev/null" } ' if test -r core; then echo 1>&2 "BAD: T.misc getline ans dropped core"; fi $awk 'BEGIN { unireghf() } function unireghf(hfeed) { hfeed[1] = 0 }' if test -r core; then echo 1>&2 "BAD: T.misc unireghf dropped core"; fi echo x | $awk '/[/]/' 2>foo grep 'nonterminated character class' foo >/dev/null || error 'BAD: T.misc nonterminated fails' if test -r core; then echo 1>&2 "BAD: T.misc nonterminated dropped core"; fi $awk ' function f() { return 12345 } BEGIN { printf "<%s>\n", f() } ' >foo grep '<12345>' foo >/dev/null || echo 'BAD: T.misc <12345> fails' echo 'abc def ghi jkl' >foo $awk ' BEGIN { RS = "" while (getline <"foo") print }' >foo1 $awk 'END {print NR}' foo1 | grep 4 >/dev/null || echo 'BAD: T.misc abcdef fails' # Test for RS regex matching an empty record at EOF echo a | $awk 1 RS='a\n' > foo1 cat << \EOF > foo2 EOF diff foo1 foo2 || echo 'BAD: T.misc RS regex matching an empty record at EOF fails' # Test for RS regex being reapplied echo aaa1a2a | $awk 1 RS='^a' >foo1 cat << \EOF > foo2 aa1a2a EOF diff foo1 foo2 || echo 'BAD: T.misc ^regex reapplied fails' # ^-anchored RS matching should be active at the start of each input file tee foo1 foo2 >foo3 << \EOF aaa EOF $awk 1 RS='^a' foo1 foo2 foo3 >foo4 cat << \EOF > foo5 aa aa aa EOF diff foo4 foo5 || echo 'BAD: T.misc ^RS matches the start of every input file fails' # The following should not produce a warning about changing a constant # nor about a curdled tempcell list $awk 'function f(x) { x = 2 } BEGIN { f(1) }' >foo grep '^' foo && echo 'BAD: test constant change fails' # The following should not produce a warning about a curdled tempcell list $awk 'function f(x) { x } BEGIN { f(1) }' >foo grep '^' foo && echo 'BAD: test tempcell list fails' $awk 'BEGIN { print 9, a=10, 11; print a; exit }' >foo1 echo '9 10 11 10' >foo2 diff foo1 foo2 || echo 'BAD: T.misc (embedded expression)' echo "abc defgh ijkl" | $awk ' { $1 = ""; line = $0; print line; print $0; $0 = line; print $0 }' >foo1 echo " defgh ijkl defgh ijkl defgh ijkl" >foo2 diff foo1 foo2 || echo 'BAD: T.misc (assignment to $0)' $awk ' function min(a, b) { if (a < b) return a else return b } BEGIN { exit } ' if test -r core; then echo 1>&2 "BAD: T.misc function min dropped core"; fi # The following should not give a syntax error message: $awk ' function expand(chart) { getline chart < "CHAR.ticks" } ' >foo grep '^' foo >/dev/null && echo 'BAD: T.misc expand error' $awk 'BEGIN { print 1e40 }' >/dev/null if test -r core; then echo 1>&2 "BAD: T.misc 1E40 dropped core"; fi # The following syntax error should not dump core: $awk ' $NF==3 {first=1} $NF==2 && first==0 && (abs($1-o1)>120||abs($2-o2)>120) {print $0} $NF==2 {o1=%1; o2=$2; first=0} ' 2>/dev/null if test -r core; then echo 1>&2 "BAD: T.misc first/abs dropped core"; fi # The following syntax error should not dump core: $awk '{ n = split($1, address, !); print address[1] }' 2>foo grep 'illegal statement' foo >/dev/null || echo 'BAD: T.misc split error' if test -r core; then echo 1>&2 "BAD: T.misc split! dropped core"; fi # The following should cause a syntax error message $awk 'BEGIN {"hello"}' 2>foo grep 'illegal statement' foo >/dev/null || echo 'BAD: T.misc hello error' # The following should give a syntax error message: $awk ' function pile(c, r) { r = ++pile[c] } { pile($1) } ' 2>foo grep 'context is' foo >/dev/null || echo 'BAD: T.misc pile error' # This should complain about missing atan2 argument: $awk 'BEGIN { atan2(1) }' 2>foo grep 'requires two arg' foo >/dev/null || echo 'BAD: T.misc atan2 error' # This should not core dump: $awk 'BEGIN { f() } function f(A) { delete A[1] } ' if test -r core; then echo 1>&2 "BAD: T.misc delete dropped core"; fi # nasty one: should not be able to overwrite constants $awk 'BEGIN { gsub(/ana/,"anda","banana") printf "the monkey ate a %s\n", "banana" } ' >/dev/null 2>foo grep 'syntax error' foo >/dev/null || echo 'BAD: T.misc gsub banana error' # nasty one: should not be able to overwrite constants $awk 'BEGIN { sub(/ana/,"anda","banana") printf "the monkey ate a %s\n", "banana" } ' >/dev/null 2>foo grep 'syntax error' foo >/dev/null || echo 'BAD: T.misc sub banana error' # line numbers used to double-count comments $awk '# # # /x ' >/dev/null 2>foo grep 'line [45]' foo >/dev/null || echo 'BAD: T.misc lineno' echo 'x  \y' >foo1 $awk 'BEGIN { print "x\f\r\b\v\a\\y" }' >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc weird chars' echo 0 >foo1 $awk ' BEGIN { exit } { print } END { print NR }' >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc BEGIN exit' echo 1 >foo1 $awk ' { exit } END { print NR }' /etc/passwd >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc immmediate exit' echo 1 >foo1 $awk ' {i = 1; while (i <= NF) {if (i == NF) exit; i++ } } END { print NR }' /etc/passwd >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc immmediate exit 2' echo 1 >foo1 $awk ' function f() { i = 1; while (i <= NF) {if (i == NF) return NR; i++ } } { if (f() == 1) exit } END { print NR }' /etc/passwd >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc while return' echo 1 >foo1 $awk ' function f() { split("a b c", arr) for (i in arr) {if (i == 3) return NR; i++ } } { if (f() == 1) exit } END { print NR }' /etc/passwd >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc while return' echo 1 >foo1 $awk ' {i = 1; do { if (i == NF) exit; i++ } while (i <= NF) } END { print NR }' /etc/passwd >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc immmediate exit 3' echo 1 >foo1 $awk ' function f() { i = 1; do { if (i == NF) return NR; i++ } while (i <= NF) } { if (f() == 1) exit } END { print NR }' /etc/passwd >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc do return' echo 1 >foo1 $awk ' {i = 1; do { if (i == NF) break; i++ } while (i <= NF); exit } END { print NR }' /etc/passwd >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc immmediate exit 4' echo 1 >foo1 $awk ' { n = split($0, x) for (i in x) { if (i == 1) exit } } END { print NR }' /etc/passwd >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc immmediate exit 5' echo XXXXXXXX >foo1 $awk 'BEGIN { s = "ab\fc\rd\be" t = s; gsub("[" s "]", "X", t); print t }' >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc weird escapes in char class' $awk '{}' /etc/passwd glop/glop >foo 2>foo2 grep "can't open.*glop" foo2 >/dev/null || echo "BAD: T.misc can't open" echo ' a aa b c ' >foo echo 3 >foo1 $awk 'BEGIN { RS = "" }; END { print NR }' foo >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc RS botch' $awk 'BEGIN \ { print "hello, world" } }}}' >foo1 2>foo2 grep 'source line 4' foo2 >/dev/null 2>&1 || echo 'BAD: T.misc continuation line number' echo 111 222 333 >foo $awk '{ f[1]=1; f[2]=2; print $f[1], $f[1]++, $f[2], f[1], f[2] }' foo >foo2 echo 111 111 222 2 2 >foo1 cmp -s foo1 foo2 || echo 'BAD: T.misc $f[1]++' # These should be syntax errors $awk . 2>foo grep "syntax error" foo >/dev/null || echo 1>&2 "BAD: T.misc syntax error . fails" $awk .. 2>foo grep "syntax error" foo >/dev/null || echo 1>&2 "BAD: T.misc syntax error .. fails" $awk .E. 2>foo grep "syntax error" foo >/dev/null || echo 1>&2 "BAD: T.misc syntax error .E. fails" $awk .++. 2>foo grep "syntax error" foo >/dev/null || echo 1>&2 "BAD: T.misc syntax error .++. fails" # These should be syntax errors $awk '$' 2>foo grep "unexpected" foo >/dev/null || echo 1>&2 "BAD: T.misc syntax error $ fails" $awk '{print $' 2>foo grep "unexpected" foo >/dev/null || echo 1>&2 "BAD: T.misc syntax error $2 fails" $awk '"' 2>foo grep "non-terminated" foo >/dev/null || echo 1>&2 "BAD: T.misc bare quote fails" # %c of 0 is explicit null byte ./echo '3' >foo1 $awk 'BEGIN {printf("%c%c\n", 0, 0) }' | wc | $awk '{print $3}' >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc null byte' # non-terminated RE $awk /xyz >foo 2>&1 grep "non-terminated" foo >/dev/null || echo 1>&2 "BAD: T.misc non-terminated RE" # next several were infinite loops, found by brian tsang. # this is his example: $awk 'BEGIN { switch (substr("x",1,1)) { case /ask.com/: break case "google": break } }' >foo 2>&1 grep "illegal statement" foo >/dev/null || echo 1>&2 "BAD: T.misc looping syntax error 1" $awk 'BEGIN { s { c /./ } }' >foo 2>&1 grep "illegal statement" foo >/dev/null || echo 1>&2 "BAD: T.misc looping syntax error 2" $awk 'BEGIN { s { c /../ } }' >foo 2>&1 grep "illegal statement" foo >/dev/null || echo 1>&2 "BAD: T.misc looping syntax error 3" $awk 'BEGIN {printf "%2$s %1$s\n", "a", "b"}' >foo 2>&1 grep "'$' not permitted in awk formats" foo >/dev/null || echo 1>&2 "BAD: T.misc '$' not permitted in formats" echo 'a b c de fg hi' >foo0 $awk 'END { print NF, $0 }' foo0 >foo1 awk '{ print NF, $0 }' foo0| tail -1 >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc END must preserve $0' echo 'fg hi' >foo0 $awk 'END { print NF, $0 }' foo0 >foo1 awk '{ print NF, $0 }' foo0| tail -1 >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc END must preserve $0' echo '' >foo0 $awk 'END { print NF, $0 }' foo0 >foo1 awk '{ print NF, $0 }' foo0| tail -1 >foo2 cmp -s foo1 foo2 || echo 'BAD: T.misc END must preserve $0' # Check for nonzero exit status on I/O error. echo 'E 2' >foo1 (trap '' PIPE; "$awk" 'BEGIN { print "hi"; }' 2>/dev/null; echo "E $?" >foo2) | : cmp -s foo1 foo2 || echo 'BAD: T.misc exit status on I/O error' + +# Check for clobbering of the lexer's regular expression buffer. +# If the output is "a1" instead of "1b", /b/ clobbered /a/. +echo 1b >foo1 +echo ab | $awk '{ sub(/a/, "b" ~ /b/); print }' >foo2 +cmp -s foo1 foo2 || echo 'BAD: T.misc lexer regex buffer clobbered' + +# Check handling of octal \OOO and hex \xHH esc. seqs. in strings. +echo 'hello888 +hello +hello +helloxGOO +hello +0A' > foo1 +$awk 'BEGIN { print "hello\888" }' > foo2 +$awk 'BEGIN { print "hello\x000A" }' >> foo2 +$awk 'BEGIN { printf "hello\x0A" }' >> foo2 +$awk 'BEGIN { print "hello\xGOO" }' >> foo2 +$awk 'BEGIN { print "hello\x0A0A" }' >> foo2 +cmp -s foo1 foo2 || echo '�BAD: T.misc escape sequences in strings mishandled' diff --git a/contrib/one-true-awk/testdir/T.overflow b/contrib/one-true-awk/testdir/T.overflow index d3d97d4cc57d..ac9c0bdde9d8 100755 --- a/contrib/one-true-awk/testdir/T.overflow +++ b/contrib/one-true-awk/testdir/T.overflow @@ -1,86 +1,88 @@ echo T.overflow: test some overflow conditions awk=${awk-../a.out} $awk 'BEGIN { for (i = 0; i < 1000; i++) printf("abcdefghijklmnopqsrtuvwxyz") printf("\n") exit }' >foo1 $awk '{print}' foo1 >foo2 cmp -s foo1 foo2 || echo 'BAD: T.overflow record 1' echo 'abcdefghijklmnopqsrtuvwxyz' >foo1 echo hello | $awk ' { for (i = 1; i < 500; i++) s = s "abcdefghijklmnopqsrtuvwxyz " $0 = s print $1 }' >foo2 cmp -s foo1 foo2 || echo 'BAD: T.overflow abcdef' # default input record 3072, fields 200: $awk ' BEGIN { for (j = 0; j < 2; j++) { for (i = 0; i < 500; i++) printf(" 123456789") printf("\n"); } } ' >foo1 $awk '{$1 = " 123456789"; print}' foo1 >foo2 cmp -s foo1 foo2 || echo 'BAD: T.overflow -mr -mf set $1' $awk ' BEGIN { for (j = 0; j < 2; j++) { for (i = 0; i < 500; i++) printf(" 123456789") printf("\n"); } } ' >foo $awk '{print NF}' foo >foo1 echo '500 500' >foo2 cmp -s foo1 foo2 || echo 'BAD: T.overflow -mr -mf NF' rm -f core # this should not drop core $awk 'BEGIN { for (i = 1; i < 1000; i++) s = s "a-z" if ("x" ~ "[" s "]") print "ugh" }' >foo 2>foo test -r core && echo 1>&2 "BAD: T.overflow too long char class dropped core" echo 4000004 >foo1 $awk ' BEGIN { x1 = sprintf("%1000000s\n", "hello") x2 = sprintf("%-1000000s\n", "world") x3 = sprintf("%1000000.1000000s\n", "goodbye") x4 = sprintf("%-1000000.1000000s\n", "goodbye") print length(x1 x2 x3 x4) }' >foo2 cmp -s foo1 foo2 || echo 'BAD: T.overflow huge sprintfs' echo 0 >foo1 $awk ' BEGIN { for (i = 0; i < 100000; i++) x[i] = i for (i in x) delete x[i] n = 0 for (i in x) n++ print n }' >foo2 cmp -s foo1 foo2 || echo 'BAD: T.overflow big array' echo x >foo1 $awk '{print $40000000000000}' foo2 2>foo grep "out of range field" foo >/dev/null || echo 1>&2 "BAD: T.overflow \$400000" rm -rf /tmp/awktestfoo* $awk 'BEGIN { for (i=1; i <= 1000; i++) print i >("/tmp/awktestfoo" i) }' ls /tmp/awktestfoo* | grep '1000' >/dev/null || echo 1>&2 "BAD: T.overflow openfiles" +rm -rf /tmp/awktestfoo* +exit 0 diff --git a/contrib/one-true-awk/testdir/T.split b/contrib/one-true-awk/testdir/T.split index f7b24ba52d75..d938404b796b 100755 --- a/contrib/one-true-awk/testdir/T.split +++ b/contrib/one-true-awk/testdir/T.split @@ -1,224 +1,225 @@ #!/bin/sh awk=${awk-../a.out} WORKDIR=$(mktemp -d /tmp/nawktest.XXXXXX) TEMP0=$WORKDIR/test.temp.0 TEMP1=$WORKDIR/test.temp.1 TEMP2=$WORKDIR/test.temp.2 RESULT=0 fail() { echo "$1" >&2 RESULT=1 } echo T.split: misc tests of field splitting and split command $awk 'BEGIN { # Assign string to $0, then change FS. FS = ":" $0="a:bc:def" FS = "-" print FS, $1, NF # Assign number to $0, then change FS. FS = "2" $0=1212121 FS="3" print FS, $1, NF }' > $TEMP1 echo '- a 3 3 1 4' > $TEMP2 diff $TEMP1 $TEMP2 || fail 'BAD: T.split 0.1' $awk 'BEGIN { # FS changes after getline. FS = ":" "echo a:bc:def" | getline FS = "-" print FS, $1, NF }' > $TEMP1 echo '- a 3' > $TEMP2 diff $TEMP1 $TEMP2 || fail 'BAD: T.split 0.2' echo ' a a:b c:d:e e:f:g:h' > $TEMP0 $awk 'BEGIN { FS = ":" while (getline <"'$TEMP0'" > 0) print NF }' > $TEMP1 echo '0 1 2 3 4' > $TEMP2 diff $TEMP1 $TEMP2 || fail 'BAD: T.split 0.3' # getline var shouldn't impact fields. echo 'f b a' > $TEMP0 $awk '{ FS = ":" getline a < "/etc/passwd" print $1 }' $TEMP0 > $TEMP1 echo 'f' > $TEMP2 diff $TEMP1 $TEMP2 || fail 'BAD: T.split 0.4' echo 'a b c d foo e f g h i bar' > $TEMP0 $awk '{ FS=":" getline v print $2, NF FS=" " }' $TEMP0 > $TEMP1 echo 'b 4 f 5' > $TEMP2 diff $TEMP1 $TEMP2 || fail 'BAD: T.split 0.5' echo 'a.b.c=d.e.f g.h.i=j.k.l m.n.o=p.q.r' > $TEMP0 echo 'b h n' > $TEMP1 $awk 'BEGIN { FS="=" } { FS="."; $0=$1; print $2; FS="="; }' $TEMP0 > $TEMP2 diff $TEMP1 $TEMP2 || fail 'BAD: T.split (record assignment 1)' echo 'a.b.c=d.e.f g.h.i=j.k.l m.n.o=p.q.r' > $TEMP0 echo 'd.e.f b j.k.l h p.q.r n' > $TEMP1 $awk 'BEGIN { FS="=" } { print $2; FS="."; $0=$1; print $2; FS="="; }' $TEMP0 > $TEMP2 diff $TEMP1 $TEMP2 || fail 'BAD: T.split (record assignment 2)' echo 'abc de f ' > $TEMP0 who | sed 10q >> $TEMP0 sed 10q /etc/passwd >> $TEMP0 $awk ' { n = split($0, x, "") m = length($0) if (m != n) print "error 1", NR s = "" for (i = 1; i <= m; i++) s = s x[i] if (s != $0) print "error 2", NR print s }' $TEMP0 > $TEMP1 diff $TEMP0 $TEMP1 || fail 'BAD: T.split 1' # assumes same test.temp.0! bad design $awk ' { n = split($0, x, //) m = length($0) if (m != n) print "error 1", NR s = "" for (i = 1; i <= m; i++) s = s x[i] if (s != $0) print "error 2", NR print s }' $TEMP0 > $TEMP1 diff $TEMP0 $TEMP1 || fail 'BAD: T.split //' $awk ' BEGIN { FS = "" } { n = split($0, x) # will be split with FS m = length($0) if (m != n) print "error 1", NR s = "" for (i = 1; i <= m; i++) s = s x[i] if (s != $0) print "error 2", NR print s }' $TEMP0 > $TEMP2 diff $TEMP0 $TEMP2 || fail 'BAD: T.split 2' # assumes same test.temp.0! $awk ' BEGIN { FS = "" } { n = NF m = length($0) if (m != n) print "error 1", NR s = "" for (i = 1; i <= m; i++) s = s $i if (s != $0) print "error 2", NR print s }' $TEMP0 > $TEMP2 diff $TEMP0 $TEMP2 || fail 'BAD: T.split 3' $awk ' { n = split( $0, temp, /^@@@ +/ ) print n }' > $TEMP1 < $TEMP2 diff $TEMP1 $TEMP2 || fail 'BAD: T.split 4' rm -f $WORKDIR/test.temp* echo ' a bc def' > $TEMP0 $awk ' { print split($0, x, "") }' $TEMP0 > $TEMP1 echo '0 1 2 3' > $TEMP2 diff $TEMP1 $TEMP2 || fail 'BAD: T.split null 3rd arg' rm -f $WORKDIR/test.temp* $awk 'BEGIN { a[1]="a b" print split(a[1],a),a[1],a[2] }' > $TEMP1 echo '2 a b' > $TEMP2 diff $TEMP1 $TEMP2 || fail 'BAD: T.split(a[1],a)' $awk 'BEGIN { a = "cat\n\n\ndog" split(a, b, "[\r\n]+") print b[1], b[2] }' > $TEMP1 echo 'cat dog' > $TEMP2 diff $TEMP1 $TEMP2 || fail 'BAD: T.split(a, b, "[\r\n]+")' +rm -rf $WORKDIR exit $RESULT diff --git a/contrib/one-true-awk/testdir/T.utf b/contrib/one-true-awk/testdir/T.utf new file mode 100755 index 000000000000..18f2b9c355cf --- /dev/null +++ b/contrib/one-true-awk/testdir/T.utf @@ -0,0 +1,194 @@ +#!/bin/sh + +echo T.utf: tests of utf functions + +awk=${awk-../a.out} + +$awk ' +BEGIN { + FS = "\t" + awk = "../a.out" +} +NF == 0 || $1 ~ /^#/ { + next +} +$1 ~ /try/ { # new test + nt++ + sub(/try [a-zA-Z_0-9]+ /, "") + prog = $0 + printf("try %3d %s\n", nt, prog) + prog = sprintf("%s -F\"\\t\" '"'"'%s'"'"'", awk, prog) + # print "prog is", prog + nt2 = 0 + while (getline > 0) { + if (NF == 0) # blank line terminates a sequence + break + input = $1 + for (i = 2; i < NF; i++) # input data + input = input "\t" $i + test = sprintf("./echo '"'"'%s'"'"' | %s >foo1; ", + input, prog) + if ($NF == "\"\"") + output = ">foo2;" + else + output = sprintf("./echo '"'"'%s'"'"' >foo2; ", $NF) + gsub(/\\t/, "\t", output) + gsub(/\\n/, "\n", output) + run = sprintf("diff foo1 foo2 || echo test %d.%d failed", + nt, ++nt2) + # print "input is", input + # print "test is", test + # print "output is", output + # print "run is", run + system(test output run) + } + tt += nt2 +} +END { print tt, "tests" } +' <<\!!!! +# General format: +# try program as rest of line +# $1 $2 $3 output1 (\t for tab, \n for newline, +# $1 $2 $3 output2 ("" for null) +# ... terminated by blank line + +# try another program... + +try length { print length($1) } + 0 +a 1 +の今がその時だ 7 +Сейчас 6 +现在是时候了 6 +给所有的好男 6 +来参加聚会。 6 +😀 1 +🖕 finger 8 +Τωρα 4 +για 3 +να 2 +עכשיו 5 +לכל 3 +לבוא 4 +の今がその時だ 7 +지금이 3 +모든 2 +파티에 3 +Сейчас 6 +для 3 +прийти 6 + +try index { print index($1, $2) } +abc a 1 +abc b 2 +abc x 0 +现在是时候了 "" 0 +现在是时候了 了 6 +现在是时候了 在是 2 +现在是时候了 x 0 +现x在是时候了 x 2 +🖕 fingerすべての善人のためにすべての善人のために f 3 +🖕 finger🖕 r🖕 8 + +try substr { print substr($0, 2, 3) } +abcdef bcd +Τωρα ειναι η ωρα +Τω ω +지금 이절호의 금 이 +xпyрийти пyр + +try rematch { print $1 ~ $2 } +abc a 1 +abc x 0 +すべての善人のために の 1 +すべての善人のために の.*の 1 +すべての善人のために の.*て 0 +Τωρα ω+ 1 + +# replace first occurrence of $2 by $3 in $1 +try sub { n = sub($2, $3, $1); print n, $1 } +abcdef bc XYZ 1 aXYZdef +abcdef xy XYZ 0 abcdef +の今がその時だ の NO 1 NO今がその時だ +🖕 finger 🖕.*g FING 1 FINGer +Сейчас . x 1 xейчас + +# replace all occurrences of $2 by $3 in $1 +try gsub { n = gsub($2, $3, $1); print n, $1 } +abcdef bc XYZ 1 aXYZdef +abcdef xy XYZ 0 abcdef +の今がその時だ の NO 2 NO今がそNO時だ +🖕 finger 🖕.*g FING 1 FINGer +Сейчас . x 6 xxxxxx + +try match { print match($1, $2), RSTART, RLENGTH } +abc [^a] 2 2 1 +abc [^ab] 3 3 1 +すべての善人のために [^す] 2 2 1 +すべての善人のために [^ぁ-ゖ] 5 5 1 +abc a 1 1 1 +abc x 0 0 -1 +すべての善人のために の 4 4 1 +すべての善人のために の.*の 4 4 4 +すべての善人のために の.*て 0 0 -1 +Τωρα ω+ 2 2 1 +Τωρα x+ 0 0 -1 +Τωρα ω. 2 2 2 +すべての善人のために [の] 4 4 1 +すべての善人のために [ぁ-え] 0 0 -1 +すべての善人のために [^ぁ-え] 1 1 1 +Τωρα ειναι η [α-ω] 2 2 1 +Τωρα ειναι η [α-ω]+ 2 2 3 +xxxΤωρα ειναι η [Α-Ω] 4 4 1 +για όλους τους καλούς ά α.*α 3 3 15 +να έρθει στο πά [^ν] 2 2 1 + +# FS="" should split into unicode chars +try emptyFS BEGIN {FS=""} {print NF} +すべての善人のために 10 +の今がその時だ 7 +Сейчас 6 +现在是时候了 6 +给所有的好男 6 +来参加聚会。 6 +😀 1 +🖕 finger 8 + +# printf(%N.Ns) for utf8 strings +try printfs1 {printf("[%5.2s][%-5.2s]\n"), $1, $1} +abcd [ ab][ab ] +现在abc [ 现在][现在 ] +现ωabc [ 现ω][现ω ] +ωabc [ ωa][ωa ] +Сейчас [ Се][Се ] +Сейxyz [ Се][Се ] +😀 [ 😀][😀 ] + +# printf(%N.Ns) for utf8 strings +try printfs2 {printf("[%5s][%-5s]\n"), $1, $1} +abcd [ abcd][abcd ] +现在ab [ 现在ab][现在ab ] +a现在ab [a现在ab][a现在ab] +a现在abc [a现在abc][a现在abc] +现ωab [ 现ωab][现ωab ] +ωabc [ ωabc][ωabc ] +Сейчас [Сейчас][Сейчас] +😀 [ 😀][😀 ] + +# printf(%N.Ns) for utf8 strings +try printfs3 {printf("[%.2s][%-.2s]\n"), $1, $1} +abcd [ab][ab] +现在abc [现在][现在] +现ωabc [现ω][现ω] +ω [ω][ω] +😀 [😀][😀] + +# printf(%c) for utf +try printfc {printf("%c %c\n", $1, substr($1,2,1))} +すべての善人のために す べ +の今がその時だ の 今 +Сейчас С е +现在是时候了 现 在 +😀🖕 😀 🖕 + +!!!! diff --git a/contrib/one-true-awk/testdir/T.utfre b/contrib/one-true-awk/testdir/T.utfre new file mode 100755 index 000000000000..20e66cbde9a5 --- /dev/null +++ b/contrib/one-true-awk/testdir/T.utfre @@ -0,0 +1,234 @@ +echo T.utfre: tests of regular expression code for Unicode/utf-8 +# adapted from T.re + +awk ' +BEGIN { + FS = "\t" + awk = "../a.out" +} +NF == 0 { + next +} +$1 != "" { # new test + re = $1 +} +$2 != "" { # either ~ or !~ + op = $2 + if (op == "~") + neg = "!" + else if (op == "!~") + neg = "" +} +$3 != "" { # new test string + str = $3 +} +$3 == "\"\"" { # explicit empty line + $3 = "" +} +NF > 2 { # generate a test + input = $3 + test = sprintf("./echo '"'"'%s'"'"' | %s '"'"'%s/%s/ {print \"%d fails %s %s %s\"}'"'"'", + input, awk, neg, re, NR, re, op, input) + # printf(" %3d %s %s %s:\n", NR, re, op, input) + # print "test is |" test "|" + system(test) + # system("bprint -c ../a.out") + nt++ +} +END { print " " nt, "tests" } +' <<\!!!! + ~ 🖕 + 🖕🖕 + 🖕🖕🖕 + "" +🖕 ~ 🖕 + b🖕 + b🖕b + !~ "" + 时 + xxxxx +. ~ 时 + x时x + 🙂 + !~ "" +.の ~ xの + xxの + xのx + !~ の + のx + "" +$ ~ x + 🙂 + "" +.$ ~ 모 + xx모 + x모x + !~ "" +д$ ~ д + bд + bbbд + !~ дb + x + "" +^ ~ и + "" + ^ +^λ$ ~ λ + !~ xλ + λx + xλx + "" +^λ.$ ~ λx + λλ + !~ xλ + λλλ + λxy + "" +^$ ~ "" + !~ に + ^ +^.해 ~ め해 + め해해 + !~ 해 + "" +^.*해 ~ 해 + め해 + めめめめめめ해 + !~ "" +^.+해 ~ め해 + めめめめめめ해 + !~ "" + 해 + 해め +해* ~ "" + 해 + 해해해해 + め해 + めめめめ +해해* ~ 해 + 해해해 + め해 + !~ めめめめ + "" +\$ ~ 🖕$ + $ + $🖕 + 🖕$🖕 + !~ "" + 🖕 +\. ~ . + !~ 🖕 + "" +xθ+y ~ xθy + xθθy + xθθθθθθy + !~ θy + xy + xθ +xθ?y ~ xy + xθy + !~ xθθy +θ?b?の? ~ "" + x +^a?b?め ~ め + aめ + bめ + abめ + めa + !~ "" + ab + aba +[Α-Ω] ~ Α + aΔb + xyΩ + !~ abc + β + "" +[^Α-Ω] ~ δ + aΔb + xyΩ + !~ Α + Δ + "" +[Α-ΔΦ-Ω] ~ Α + Β + Δ + Φ + Ω + !~ Π + Σ +Π[[:lower:]]+ ~ Πa + Πab + !~ Π + ΠX + Π: + Π[ + Π] +の[0-9]+に ~ の0に + の23に + の12345に + !~ 0に + のに +の[0-9]?に ~ のに + の1に + !~ の23に +の[[]に ~ の[に + !~ のに + の[[]に + の]に +の[[-]に ~ の[に + の-に + !~ のに + の[[]に + の]に +の[[-a]に ~ の[に + のaに + の]に + !~ のに + の[[]に + の-に +の[]-a]に ~ の]に + のaに + !~ のに + の[に + の-に +の[]]に ~ の]に + !~ のに + の[]]に + の[に +の[^[]に ~ のaに + !~ の[に +の[-]に ~ の-に + !~ のに + の+に +の[^-]に ~ の+に + !~ の-に + のに +の[][]に ~ の[に + の]に + !~ のに + の][に + の[]に +の[z-a]に ~ のに + !~ の + に + のaに + のzに + の-に +に|だ ~ だ + に + だに + !~ a + "" +^στο|τους$ ~ στο + στοd + aτους + τους + !~ xστο + τουςa +^(στο|τους)$ ~ στο + τους + !~ στοτους + στοx + cτους +!!!! diff --git a/contrib/one-true-awk/testdir/tt.15 b/contrib/one-true-awk/testdir/tt.15 index c05e61f5aa12..cd48064f0d6c 100644 --- a/contrib/one-true-awk/testdir/tt.15 +++ b/contrib/one-true-awk/testdir/tt.15 @@ -1,33 +1,33 @@ # fmt - format # input: text # output: text formatted into lines of <= 72 characters BEGIN { maxlen = 72 } /^[ \t]/ { printline(); print; next } # verbatim ###/^ +/ { printline(); } # whitespace == break /./ { for (i = 1; i <= NF; i++) addword($i); next } /^$/ { printline(); print "" } END { printline() } function addword(w) { - ## print "adding [", w, "] ", length(w), length(line), maxlen + print "adding [", w, "] ", length(w), length(line), maxlen if (length(line) + length(w) > maxlen) printline() if (length(w) > 2 && ( w ~ /[\.!]["?)]?$/ || w ~ /[?!]"?$/) && w !~ /^(Mr|Dr|Ms|Mrs|vs|Ph.D)\.$/) w = w " " line = line " " w } function printline() { if (length(line) > 0) { sub(/ +$/, "", line) print substr(line, 2) # removes leading blank line = "" } } diff --git a/contrib/one-true-awk/tran.c b/contrib/one-true-awk/tran.c index c6ae890c8a91..482eede1de4c 100644 --- a/contrib/one-true-awk/tran.c +++ b/contrib/one-true-awk/tran.c @@ -1,658 +1,648 @@ /**************************************************************** Copyright (C) Lucent Technologies 1997 All Rights Reserved Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that the copyright notice and this permission notice and warranty disclaimer appear in supporting documentation, and that the name Lucent Technologies or any of its entities not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. LUCENT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL LUCENT OR ANY OF ITS ENTITIES BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ****************************************************************/ #define DEBUG #include #include #include #include #include #include "awk.h" #define FULLTAB 2 /* rehash when table gets this x full */ #define GROWTAB 4 /* grow table by this factor */ Array *symtab; /* main symbol table */ char **FS; /* initial field sep */ char **RS; /* initial record sep */ char **OFS; /* output field sep */ char **ORS; /* output record sep */ char **OFMT; /* output format for numbers */ char **CONVFMT; /* format for conversions in getsval */ Awkfloat *NF; /* number of fields in current record */ Awkfloat *NR; /* number of current record */ Awkfloat *FNR; /* number of current record in current file */ char **FILENAME; /* current filename argument */ Awkfloat *ARGC; /* number of arguments from command line */ char **SUBSEP; /* subscript separator for a[i,j,k]; default \034 */ Awkfloat *RSTART; /* start of re matched with ~; origin 1 (!) */ Awkfloat *RLENGTH; /* length of same */ Cell *fsloc; /* FS */ Cell *nrloc; /* NR */ Cell *nfloc; /* NF */ Cell *fnrloc; /* FNR */ Cell *ofsloc; /* OFS */ Cell *orsloc; /* ORS */ Cell *rsloc; /* RS */ Array *ARGVtab; /* symbol table containing ARGV[...] */ Array *ENVtab; /* symbol table containing ENVIRON[...] */ Cell *rstartloc; /* RSTART */ Cell *rlengthloc; /* RLENGTH */ Cell *subseploc; /* SUBSEP */ Cell *symtabloc; /* SYMTAB */ Cell *nullloc; /* a guaranteed empty cell */ Node *nullnode; /* zero&null, converted into a node for comparisons */ Cell *literal0; extern Cell **fldtab; -static void -setfree(Cell *vp) -{ - if (&vp->sval == FS || &vp->sval == RS || - &vp->sval == OFS || &vp->sval == ORS || - &vp->sval == OFMT || &vp->sval == CONVFMT || - &vp->sval == FILENAME || &vp->sval == SUBSEP) - vp->tval |= DONTFREE; - else - vp->tval &= ~DONTFREE; -} - void syminit(void) /* initialize symbol table with builtin vars */ { literal0 = setsymtab("0", "0", 0.0, NUM|STR|CON|DONTFREE, symtab); /* this is used for if(x)... tests: */ nullloc = setsymtab("$zero&null", "", 0.0, NUM|STR|CON|DONTFREE, symtab); nullnode = celltonode(nullloc, CCON); fsloc = setsymtab("FS", " ", 0.0, STR|DONTFREE, symtab); FS = &fsloc->sval; rsloc = setsymtab("RS", "\n", 0.0, STR|DONTFREE, symtab); RS = &rsloc->sval; ofsloc = setsymtab("OFS", " ", 0.0, STR|DONTFREE, symtab); OFS = &ofsloc->sval; orsloc = setsymtab("ORS", "\n", 0.0, STR|DONTFREE, symtab); ORS = &orsloc->sval; OFMT = &setsymtab("OFMT", "%.6g", 0.0, STR|DONTFREE, symtab)->sval; CONVFMT = &setsymtab("CONVFMT", "%.6g", 0.0, STR|DONTFREE, symtab)->sval; FILENAME = &setsymtab("FILENAME", "", 0.0, STR|DONTFREE, symtab)->sval; nfloc = setsymtab("NF", "", 0.0, NUM, symtab); NF = &nfloc->fval; nrloc = setsymtab("NR", "", 0.0, NUM, symtab); NR = &nrloc->fval; fnrloc = setsymtab("FNR", "", 0.0, NUM, symtab); FNR = &fnrloc->fval; subseploc = setsymtab("SUBSEP", "\034", 0.0, STR|DONTFREE, symtab); SUBSEP = &subseploc->sval; rstartloc = setsymtab("RSTART", "", 0.0, NUM, symtab); RSTART = &rstartloc->fval; rlengthloc = setsymtab("RLENGTH", "", 0.0, NUM, symtab); RLENGTH = &rlengthloc->fval; symtabloc = setsymtab("SYMTAB", "", 0.0, ARR, symtab); free(symtabloc->sval); symtabloc->sval = (char *) symtab; } void arginit(int ac, char **av) /* set up ARGV and ARGC */ { Cell *cp; int i; char temp[50]; ARGC = &setsymtab("ARGC", "", (Awkfloat) ac, NUM, symtab)->fval; cp = setsymtab("ARGV", "", 0.0, ARR, symtab); ARGVtab = makesymtab(NSYMTAB); /* could be (int) ARGC as well */ free(cp->sval); cp->sval = (char *) ARGVtab; for (i = 0; i < ac; i++) { double result; sprintf(temp, "%d", i); if (is_number(*av, & result)) setsymtab(temp, *av, result, STR|NUM, ARGVtab); else setsymtab(temp, *av, 0.0, STR, ARGVtab); av++; } } void envinit(char **envp) /* set up ENVIRON variable */ { Cell *cp; char *p; cp = setsymtab("ENVIRON", "", 0.0, ARR, symtab); ENVtab = makesymtab(NSYMTAB); free(cp->sval); cp->sval = (char *) ENVtab; for ( ; *envp; envp++) { double result; if ((p = strchr(*envp, '=')) == NULL) continue; if( p == *envp ) /* no left hand side name in env string */ continue; *p++ = 0; /* split into two strings at = */ if (is_number(p, & result)) setsymtab(*envp, p, result, STR|NUM, ENVtab); else setsymtab(*envp, p, 0.0, STR, ENVtab); p[-1] = '='; /* restore in case env is passed down to a shell */ } } Array *makesymtab(int n) /* make a new symbol table */ { Array *ap; Cell **tp; ap = (Array *) malloc(sizeof(*ap)); tp = (Cell **) calloc(n, sizeof(*tp)); if (ap == NULL || tp == NULL) FATAL("out of space in makesymtab"); ap->nelem = 0; ap->size = n; ap->tab = tp; return(ap); } void freesymtab(Cell *ap) /* free a symbol table */ { Cell *cp, *temp; Array *tp; int i; if (!isarr(ap)) return; tp = (Array *) ap->sval; if (tp == NULL) return; for (i = 0; i < tp->size; i++) { for (cp = tp->tab[i]; cp != NULL; cp = temp) { xfree(cp->nval); if (freeable(cp)) xfree(cp->sval); temp = cp->cnext; /* avoids freeing then using */ free(cp); tp->nelem--; } tp->tab[i] = NULL; } if (tp->nelem != 0) WARNING("can't happen: inconsistent element count freeing %s", ap->nval); free(tp->tab); free(tp); } void freeelem(Cell *ap, const char *s) /* free elem s from ap (i.e., ap["s"] */ { Array *tp; Cell *p, *prev = NULL; int h; tp = (Array *) ap->sval; h = hash(s, tp->size); for (p = tp->tab[h]; p != NULL; prev = p, p = p->cnext) if (strcmp(s, p->nval) == 0) { if (prev == NULL) /* 1st one */ tp->tab[h] = p->cnext; else /* middle somewhere */ prev->cnext = p->cnext; if (freeable(p)) xfree(p->sval); free(p->nval); free(p); tp->nelem--; return; } } Cell *setsymtab(const char *n, const char *s, Awkfloat f, unsigned t, Array *tp) { int h; Cell *p; if (n != NULL && (p = lookup(n, tp)) != NULL) { DPRINTF("setsymtab found %p: n=%s s=\"%s\" f=%g t=%o\n", (void*)p, NN(p->nval), NN(p->sval), p->fval, p->tval); return(p); } p = (Cell *) malloc(sizeof(*p)); if (p == NULL) FATAL("out of space for symbol table at %s", n); p->nval = tostring(n); p->sval = s ? tostring(s) : tostring(""); p->fval = f; p->tval = t; p->csub = CUNK; p->ctype = OCELL; tp->nelem++; if (tp->nelem > FULLTAB * tp->size) rehash(tp); h = hash(n, tp->size); p->cnext = tp->tab[h]; tp->tab[h] = p; DPRINTF("setsymtab set %p: n=%s s=\"%s\" f=%g t=%o\n", (void*)p, p->nval, p->sval, p->fval, p->tval); return(p); } int hash(const char *s, int n) /* form hash value for string s */ { unsigned hashval; for (hashval = 0; *s != '\0'; s++) hashval = (*s + 31 * hashval); return hashval % n; } void rehash(Array *tp) /* rehash items in small table into big one */ { int i, nh, nsz; Cell *cp, *op, **np; nsz = GROWTAB * tp->size; np = (Cell **) calloc(nsz, sizeof(*np)); if (np == NULL) /* can't do it, but can keep running. */ return; /* someone else will run out later. */ for (i = 0; i < tp->size; i++) { for (cp = tp->tab[i]; cp; cp = op) { op = cp->cnext; nh = hash(cp->nval, nsz); cp->cnext = np[nh]; np[nh] = cp; } } free(tp->tab); tp->tab = np; tp->size = nsz; } Cell *lookup(const char *s, Array *tp) /* look for s in tp */ { Cell *p; int h; h = hash(s, tp->size); for (p = tp->tab[h]; p != NULL; p = p->cnext) if (strcmp(s, p->nval) == 0) return(p); /* found it */ return(NULL); /* not found */ } Awkfloat setfval(Cell *vp, Awkfloat f) /* set float val of a Cell */ { int fldno; f += 0.0; /* normalise negative zero to positive zero */ if ((vp->tval & (NUM | STR)) == 0) funnyvar(vp, "assign to"); if (isfld(vp)) { donerec = false; /* mark $0 invalid */ fldno = atoi(vp->nval); if (fldno > *NF) newfld(fldno); DPRINTF("setting field %d to %g\n", fldno, f); } else if (&vp->fval == NF) { donerec = false; /* mark $0 invalid */ setlastfld(f); - DPRINTF("setting NF to %g\n", f); + DPRINTF("setfval: setting NF to %g\n", f); } else if (isrec(vp)) { donefld = false; /* mark $1... invalid */ donerec = true; savefs(); } else if (vp == ofsloc) { if (!donerec) recbld(); } if (freeable(vp)) xfree(vp->sval); /* free any previous string */ vp->tval &= ~(STR|CONVC|CONVO); /* mark string invalid */ vp->fmt = NULL; vp->tval |= NUM; /* mark number ok */ if (f == -0) /* who would have thought this possible? */ f = 0; DPRINTF("setfval %p: %s = %g, t=%o\n", (void*)vp, NN(vp->nval), f, vp->tval); return vp->fval = f; } void funnyvar(Cell *vp, const char *rw) { if (isarr(vp)) FATAL("can't %s %s; it's an array name.", rw, vp->nval); if (vp->tval & FCN) FATAL("can't %s %s; it's a function.", rw, vp->nval); WARNING("funny variable %p: n=%s s=\"%s\" f=%g t=%o", (void *)vp, vp->nval, vp->sval, vp->fval, vp->tval); } char *setsval(Cell *vp, const char *s) /* set string val of a Cell */ { char *t; int fldno; Awkfloat f; DPRINTF("starting setsval %p: %s = \"%s\", t=%o, r,f=%d,%d\n", (void*)vp, NN(vp->nval), s, vp->tval, donerec, donefld); if ((vp->tval & (NUM | STR)) == 0) funnyvar(vp, "assign to"); + if (CSV && (vp == rsloc)) + WARNING("danger: don't set RS when --csv is in effect"); + if (CSV && (vp == fsloc)) + WARNING("danger: don't set FS when --csv is in effect"); if (isfld(vp)) { donerec = false; /* mark $0 invalid */ fldno = atoi(vp->nval); if (fldno > *NF) newfld(fldno); DPRINTF("setting field %d to %s (%p)\n", fldno, s, (const void*)s); } else if (isrec(vp)) { donefld = false; /* mark $1... invalid */ donerec = true; savefs(); } else if (vp == ofsloc) { if (!donerec) recbld(); } t = s ? tostring(s) : tostring(""); /* in case it's self-assign */ if (freeable(vp)) xfree(vp->sval); - vp->tval &= ~(NUM|CONVC|CONVO); + vp->tval &= ~(NUM|DONTFREE|CONVC|CONVO); vp->tval |= STR; vp->fmt = NULL; - setfree(vp); DPRINTF("setsval %p: %s = \"%s (%p) \", t=%o r,f=%d,%d\n", (void*)vp, NN(vp->nval), t, (void*)t, vp->tval, donerec, donefld); vp->sval = t; if (&vp->fval == NF) { donerec = false; /* mark $0 invalid */ f = getfval(vp); setlastfld(f); - DPRINTF("setting NF to %g\n", f); + DPRINTF("setsval: setting NF to %g\n", f); } return(vp->sval); } Awkfloat getfval(Cell *vp) /* get float val of a Cell */ { if ((vp->tval & (NUM | STR)) == 0) funnyvar(vp, "read value of"); if (isfld(vp) && !donefld) fldbld(); else if (isrec(vp) && !donerec) recbld(); if (!isnum(vp)) { /* not a number */ double fval; bool no_trailing; if (is_valid_number(vp->sval, true, & no_trailing, & fval)) { vp->fval = fval; if (no_trailing && !(vp->tval&CON)) vp->tval |= NUM; /* make NUM only sparingly */ } else vp->fval = 0.0; } DPRINTF("getfval %p: %s = %g, t=%o\n", (void*)vp, NN(vp->nval), vp->fval, vp->tval); return(vp->fval); } static const char *get_inf_nan(double d) { if (isinf(d)) { return (d < 0 ? "-inf" : "+inf"); } else if (isnan(d)) { return (signbit(d) != 0 ? "-nan" : "+nan"); } else return NULL; } static char *get_str_val(Cell *vp, char **fmt) /* get string val of a Cell */ { char s[256]; double dtemp; const char *p; if ((vp->tval & (NUM | STR)) == 0) funnyvar(vp, "read value of"); if (isfld(vp) && ! donefld) fldbld(); else if (isrec(vp) && ! donerec) recbld(); /* * ADR: This is complicated and more fragile than is desirable. * Retrieving a string value for a number associates the string * value with the scalar. Previously, the string value was * sticky, meaning if converted via OFMT that became the value * (even though POSIX wants it to be via CONVFMT). Or if CONVFMT * changed after a string value was retrieved, the original value * was maintained and used. Also not per POSIX. * * We work around this design by adding two additional flags, * CONVC and CONVO, indicating how the string value was * obtained (via CONVFMT or OFMT) and _also_ maintaining a copy * of the pointer to the xFMT format string used for the * conversion. This pointer is only read, **never** dereferenced. * The next time we do a conversion, if it's coming from the same * xFMT as last time, and the pointer value is different, we * know that the xFMT format string changed, and we need to * redo the conversion. If it's the same, we don't have to. * * There are also several cases where we don't do a conversion, * such as for a field (see the checks below). */ /* Don't duplicate the code for actually updating the value */ #define update_str_val(vp) \ { \ if (freeable(vp)) \ xfree(vp->sval); \ if ((p = get_inf_nan(vp->fval)) != NULL) \ strcpy(s, p); \ else if (modf(vp->fval, &dtemp) == 0) /* it's integral */ \ snprintf(s, sizeof (s), "%.30g", vp->fval); \ else \ snprintf(s, sizeof (s), *fmt, vp->fval); \ vp->sval = tostring(s); \ vp->tval &= ~DONTFREE; \ vp->tval |= STR; \ } if (isstr(vp) == 0) { update_str_val(vp); if (fmt == OFMT) { vp->tval &= ~CONVC; vp->tval |= CONVO; } else { /* CONVFMT */ vp->tval &= ~CONVO; vp->tval |= CONVC; } vp->fmt = *fmt; } else if ((vp->tval & DONTFREE) != 0 || ! isnum(vp) || isfld(vp)) { goto done; } else if (isstr(vp)) { if (fmt == OFMT) { if ((vp->tval & CONVC) != 0 || ((vp->tval & CONVO) != 0 && vp->fmt != *fmt)) { update_str_val(vp); vp->tval &= ~CONVC; vp->tval |= CONVO; vp->fmt = *fmt; } } else { /* CONVFMT */ if ((vp->tval & CONVO) != 0 || ((vp->tval & CONVC) != 0 && vp->fmt != *fmt)) { update_str_val(vp); vp->tval &= ~CONVO; vp->tval |= CONVC; vp->fmt = *fmt; } } } done: DPRINTF("getsval %p: %s = \"%s (%p)\", t=%o\n", (void*)vp, NN(vp->nval), vp->sval, (void*)vp->sval, vp->tval); return(vp->sval); } char *getsval(Cell *vp) /* get string val of a Cell */ { return get_str_val(vp, CONVFMT); } char *getpssval(Cell *vp) /* get string val of a Cell for print */ { return get_str_val(vp, OFMT); } char *tostring(const char *s) /* make a copy of string s */ { char *p = strdup(s); if (p == NULL) FATAL("out of space in tostring on %s", s); return(p); } char *tostringN(const char *s, size_t n) /* make a copy of string s */ { char *p; p = (char *) malloc(n); if (p == NULL) FATAL("out of space in tostring on %s", s); strcpy(p, s); return(p); } Cell *catstr(Cell *a, Cell *b) /* concatenate a and b */ { Cell *c; char *p; char *sa = getsval(a); char *sb = getsval(b); size_t l = strlen(sa) + strlen(sb) + 1; p = (char *) malloc(l); if (p == NULL) FATAL("out of space concatenating %s and %s", sa, sb); snprintf(p, l, "%s%s", sa, sb); l++; // add room for ' ' char *newbuf = (char *) malloc(l); if (newbuf == NULL) FATAL("out of space concatenating %s and %s", sa, sb); // See string() in lex.c; a string "xx" is stored in the symbol // table as "xx ". snprintf(newbuf, l, "%s ", p); c = setsymtab(newbuf, p, 0.0, CON|STR|DONTFREE, symtab); free(p); free(newbuf); return c; } char *qstring(const char *is, int delim) /* collect string up to next delim */ { - const char *os = is; int c, n; const uschar *s = (const uschar *) is; uschar *buf, *bp; if ((buf = (uschar *) malloc(strlen(is)+3)) == NULL) FATAL( "out of space in qstring(%s)", s); for (bp = buf; (c = *s) != delim; s++) { if (c == '\n') - SYNTAX( "newline in string %.20s...", os ); + SYNTAX( "newline in string %.20s...", is ); else if (c != '\\') *bp++ = c; else { /* \something */ c = *++s; if (c == 0) { /* \ at end */ *bp++ = '\\'; break; /* for loop */ } switch (c) { case '\\': *bp++ = '\\'; break; case 'n': *bp++ = '\n'; break; case 't': *bp++ = '\t'; break; case 'b': *bp++ = '\b'; break; case 'f': *bp++ = '\f'; break; case 'r': *bp++ = '\r'; break; case 'v': *bp++ = '\v'; break; case 'a': *bp++ = '\a'; break; default: if (!isdigit(c)) { *bp++ = c; break; } n = c - '0'; if (isdigit(s[1])) { n = 8 * n + *++s - '0'; if (isdigit(s[1])) n = 8 * n + *++s - '0'; } *bp++ = n; break; } } } *bp++ = 0; return (char *) buf; } const char *flags2str(int flags) { static const struct ftab { const char *name; int value; } flagtab[] = { { "NUM", NUM }, { "STR", STR }, { "DONTFREE", DONTFREE }, { "CON", CON }, { "ARR", ARR }, { "FCN", FCN }, { "FLD", FLD }, { "REC", REC }, { "CONVC", CONVC }, { "CONVO", CONVO }, { NULL, 0 } }; static char buf[100]; int i; char *cp = buf; for (i = 0; flagtab[i].name != NULL; i++) { if ((flags & flagtab[i].value) != 0) { if (cp > buf) *cp++ = '|'; strcpy(cp, flagtab[i].name); cp += strlen(cp); } } return buf; }