diff --git a/usr.bin/sort/bwstring.c b/usr.bin/sort/bwstring.c index cab21324c4b3..f6200c53c83e 100644 --- a/usr.bin/sort/bwstring.c +++ b/usr.bin/sort/bwstring.c @@ -1,1144 +1,1144 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (C) 2009 Gabor Kovesdan * Copyright (C) 2012 Oleg Moskalenko * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include "bwstring.h" #include "sort.h" bool byte_sort; static wchar_t **wmonths; static char **cmonths; /* initialise months */ void initialise_months(void) { const nl_item item[12] = { ABMON_1, ABMON_2, ABMON_3, ABMON_4, ABMON_5, ABMON_6, ABMON_7, ABMON_8, ABMON_9, ABMON_10, ABMON_11, ABMON_12 }; char *tmp; size_t len; - if (MB_CUR_MAX == 1) { + if (mb_cur_max == 1) { if (cmonths == NULL) { char *m; cmonths = sort_malloc(sizeof(char*) * 12); for (int i = 0; i < 12; i++) { cmonths[i] = NULL; tmp = nl_langinfo(item[i]); if (debug_sort) printf("month[%d]=%s\n", i, tmp); if (*tmp == '\0') continue; m = sort_strdup(tmp); len = strlen(tmp); for (unsigned int j = 0; j < len; j++) m[j] = toupper(m[j]); cmonths[i] = m; } } } else { if (wmonths == NULL) { wchar_t *m; wmonths = sort_malloc(sizeof(wchar_t *) * 12); for (int i = 0; i < 12; i++) { wmonths[i] = NULL; tmp = nl_langinfo(item[i]); if (debug_sort) printf("month[%d]=%s\n", i, tmp); if (*tmp == '\0') continue; len = strlen(tmp); m = sort_malloc(SIZEOF_WCHAR_STRING(len + 1)); if (mbstowcs(m, tmp, len) == ((size_t) - 1)) { sort_free(m); continue; } m[len] = L'\0'; for (unsigned int j = 0; j < len; j++) m[j] = towupper(m[j]); wmonths[i] = m; } } } } /* * Compare two wide-character strings */ static int wide_str_coll(const wchar_t *s1, const wchar_t *s2) { int ret = 0; errno = 0; ret = wcscoll(s1, s2); if (errno == EILSEQ) { errno = 0; ret = wcscmp(s1, s2); if (errno != 0) { for (size_t i = 0; ; ++i) { wchar_t c1 = s1[i]; wchar_t c2 = s2[i]; if (c1 == L'\0') return ((c2 == L'\0') ? 0 : -1); if (c2 == L'\0') return (+1); if (c1 == c2) continue; return ((int)(c1 - c2)); } } } return (ret); } /* counterparts of wcs functions */ void bwsprintf(FILE *f, struct bwstring *bws, const char *prefix, const char *suffix) { - if (MB_CUR_MAX == 1) + if (mb_cur_max == 1) fprintf(f, "%s%s%s", prefix, bws->data.cstr, suffix); else fprintf(f, "%s%S%s", prefix, bws->data.wstr, suffix); } const void* bwsrawdata(const struct bwstring *bws) { return (&(bws->data)); } size_t bwsrawlen(const struct bwstring *bws) { - return ((MB_CUR_MAX == 1) ? bws->len : SIZEOF_WCHAR_STRING(bws->len)); + return ((mb_cur_max == 1) ? bws->len : SIZEOF_WCHAR_STRING(bws->len)); } size_t bws_memsize(const struct bwstring *bws) { - return ((MB_CUR_MAX == 1) ? (bws->len + 2 + sizeof(struct bwstring)) : + return ((mb_cur_max == 1) ? (bws->len + 2 + sizeof(struct bwstring)) : (SIZEOF_WCHAR_STRING(bws->len + 1) + sizeof(struct bwstring))); } void bws_setlen(struct bwstring *bws, size_t newlen) { if (bws && newlen != bws->len && newlen <= bws->len) { bws->len = newlen; - if (MB_CUR_MAX == 1) + if (mb_cur_max == 1) bws->data.cstr[newlen] = '\0'; else bws->data.wstr[newlen] = L'\0'; } } /* * Allocate a new binary string of specified size */ struct bwstring * bwsalloc(size_t sz) { struct bwstring *ret; - if (MB_CUR_MAX == 1) + if (mb_cur_max == 1) ret = sort_malloc(sizeof(struct bwstring) + 1 + sz); else ret = sort_malloc(sizeof(struct bwstring) + SIZEOF_WCHAR_STRING(sz + 1)); ret->len = sz; - if (MB_CUR_MAX == 1) + if (mb_cur_max == 1) ret->data.cstr[ret->len] = '\0'; else ret->data.wstr[ret->len] = L'\0'; return (ret); } /* * Create a copy of binary string. * New string size equals the length of the old string. */ struct bwstring * bwsdup(const struct bwstring *s) { if (s == NULL) return (NULL); else { struct bwstring *ret = bwsalloc(s->len); - if (MB_CUR_MAX == 1) + if (mb_cur_max == 1) memcpy(ret->data.cstr, s->data.cstr, (s->len)); else memcpy(ret->data.wstr, s->data.wstr, SIZEOF_WCHAR_STRING(s->len)); return (ret); } } /* * Create a new binary string from a wide character buffer. */ struct bwstring * bwssbdup(const wchar_t *str, size_t len) { if (str == NULL) return ((len == 0) ? bwsalloc(0) : NULL); else { struct bwstring *ret; ret = bwsalloc(len); - if (MB_CUR_MAX == 1) + if (mb_cur_max == 1) for (size_t i = 0; i < len; ++i) ret->data.cstr[i] = (unsigned char) str[i]; else memcpy(ret->data.wstr, str, SIZEOF_WCHAR_STRING(len)); return (ret); } } /* * Create a new binary string from a raw binary buffer. */ struct bwstring * bwscsbdup(const unsigned char *str, size_t len) { struct bwstring *ret; ret = bwsalloc(len); if (str) { - if (MB_CUR_MAX == 1) + if (mb_cur_max == 1) memcpy(ret->data.cstr, str, len); else { mbstate_t mbs; const char *s; size_t charlen, chars, cptr; chars = 0; cptr = 0; s = (const char *) str; memset(&mbs, 0, sizeof(mbs)); while (cptr < len) { - size_t n = MB_CUR_MAX; + size_t n = mb_cur_max; if (n > len - cptr) n = len - cptr; charlen = mbrlen(s + cptr, n, &mbs); switch (charlen) { case 0: /* FALLTHROUGH */ case (size_t) -1: /* FALLTHROUGH */ case (size_t) -2: ret->data.wstr[chars++] = (unsigned char) s[cptr]; ++cptr; break; default: n = mbrtowc(ret->data.wstr + (chars++), s + cptr, charlen, &mbs); if ((n == (size_t)-1) || (n == (size_t)-2)) /* NOTREACHED */ err(2, "mbrtowc error"); cptr += charlen; } } ret->len = chars; ret->data.wstr[ret->len] = L'\0'; } } return (ret); } /* * De-allocate object memory */ void bwsfree(const struct bwstring *s) { if (s) sort_free(s); } /* * Copy content of src binary string to dst. * If the capacity of the dst string is not sufficient, * then the data is truncated. */ size_t bwscpy(struct bwstring *dst, const struct bwstring *src) { size_t nums = src->len; if (nums > dst->len) nums = dst->len; dst->len = nums; - if (MB_CUR_MAX == 1) { + if (mb_cur_max == 1) { memcpy(dst->data.cstr, src->data.cstr, nums); dst->data.cstr[dst->len] = '\0'; } else { memcpy(dst->data.wstr, src->data.wstr, SIZEOF_WCHAR_STRING(nums + 1)); dst->data.wstr[dst->len] = L'\0'; } return (nums); } /* * Copy content of src binary string to dst, * with specified number of symbols to be copied. * If the capacity of the dst string is not sufficient, * then the data is truncated. */ struct bwstring * bwsncpy(struct bwstring *dst, const struct bwstring *src, size_t size) { size_t nums = src->len; if (nums > dst->len) nums = dst->len; if (nums > size) nums = size; dst->len = nums; - if (MB_CUR_MAX == 1) { + if (mb_cur_max == 1) { memcpy(dst->data.cstr, src->data.cstr, nums); dst->data.cstr[dst->len] = '\0'; } else { memcpy(dst->data.wstr, src->data.wstr, SIZEOF_WCHAR_STRING(nums + 1)); dst->data.wstr[dst->len] = L'\0'; } return (dst); } /* * Copy content of src binary string to dst, * with specified number of symbols to be copied. * An offset value can be specified, from the start of src string. * If the capacity of the dst string is not sufficient, * then the data is truncated. */ struct bwstring * bwsnocpy(struct bwstring *dst, const struct bwstring *src, size_t offset, size_t size) { if (offset >= src->len) { dst->data.wstr[0] = 0; dst->len = 0; } else { size_t nums = src->len - offset; if (nums > dst->len) nums = dst->len; if (nums > size) nums = size; dst->len = nums; - if (MB_CUR_MAX == 1) { + if (mb_cur_max == 1) { memcpy(dst->data.cstr, src->data.cstr + offset, (nums)); dst->data.cstr[dst->len] = '\0'; } else { memcpy(dst->data.wstr, src->data.wstr + offset, SIZEOF_WCHAR_STRING(nums)); dst->data.wstr[dst->len] = L'\0'; } } return (dst); } /* * Write binary string to the file. * The output is ended either with '\n' (nl == true) * or '\0' (nl == false). */ size_t bwsfwrite(struct bwstring *bws, FILE *f, bool zero_ended) { - if (MB_CUR_MAX == 1) { + if (mb_cur_max == 1) { size_t len = bws->len; if (!zero_ended) { bws->data.cstr[len] = '\n'; if (fwrite(bws->data.cstr, len + 1, 1, f) < 1) err(2, NULL); bws->data.cstr[len] = '\0'; } else if (fwrite(bws->data.cstr, len + 1, 1, f) < 1) err(2, NULL); return (len + 1); } else { wchar_t eols; size_t printed = 0; eols = zero_ended ? btowc('\0') : btowc('\n'); while (printed < BWSLEN(bws)) { const wchar_t *s = bws->data.wstr + printed; if (*s == L'\0') { int nums; nums = fwprintf(f, L"%lc", *s); if (nums != 1) err(2, NULL); ++printed; } else { int nums; nums = fwprintf(f, L"%ls", s); if (nums < 1) err(2, NULL); printed += nums; } } fwprintf(f, L"%lc", eols); return (printed + 1); } } /* * Allocate and read a binary string from file. * The strings are nl-ended or zero-ended, depending on the sort setting. */ struct bwstring * bwsfgetln(FILE *f, size_t *len, bool zero_ended, struct reader_buffer *rb) { wint_t eols; eols = zero_ended ? btowc('\0') : btowc('\n'); - if (!zero_ended && (MB_CUR_MAX > 1)) { + if (!zero_ended && (mb_cur_max > 1)) { wchar_t *ret; ret = fgetwln(f, len); if (ret == NULL) { if (!feof(f)) err(2, NULL); return (NULL); } if (*len > 0) { if (ret[*len - 1] == (wchar_t)eols) --(*len); } return (bwssbdup(ret, *len)); - } else if (!zero_ended && (MB_CUR_MAX == 1)) { + } else if (!zero_ended && (mb_cur_max == 1)) { char *ret; ret = fgetln(f, len); if (ret == NULL) { if (!feof(f)) err(2, NULL); return (NULL); } if (*len > 0) { if (ret[*len - 1] == '\n') --(*len); } return (bwscsbdup((unsigned char*)ret, *len)); } else { *len = 0; if (feof(f)) return (NULL); if (2 >= rb->fgetwln_z_buffer_size) { rb->fgetwln_z_buffer_size += 256; rb->fgetwln_z_buffer = sort_realloc(rb->fgetwln_z_buffer, sizeof(wchar_t) * rb->fgetwln_z_buffer_size); } rb->fgetwln_z_buffer[*len] = 0; - if (MB_CUR_MAX == 1) + if (mb_cur_max == 1) while (!feof(f)) { int c; c = fgetc(f); if (c == EOF) { if (*len == 0) return (NULL); goto line_read_done; } if (c == eols) goto line_read_done; if (*len + 1 >= rb->fgetwln_z_buffer_size) { rb->fgetwln_z_buffer_size += 256; rb->fgetwln_z_buffer = sort_realloc(rb->fgetwln_z_buffer, SIZEOF_WCHAR_STRING(rb->fgetwln_z_buffer_size)); } rb->fgetwln_z_buffer[*len] = c; rb->fgetwln_z_buffer[++(*len)] = 0; } else while (!feof(f)) { wint_t c = 0; c = fgetwc(f); if (c == WEOF) { if (*len == 0) return (NULL); goto line_read_done; } if (c == eols) goto line_read_done; if (*len + 1 >= rb->fgetwln_z_buffer_size) { rb->fgetwln_z_buffer_size += 256; rb->fgetwln_z_buffer = sort_realloc(rb->fgetwln_z_buffer, SIZEOF_WCHAR_STRING(rb->fgetwln_z_buffer_size)); } rb->fgetwln_z_buffer[*len] = c; rb->fgetwln_z_buffer[++(*len)] = 0; } line_read_done: /* we do not count the last 0 */ return (bwssbdup(rb->fgetwln_z_buffer, *len)); } } int bwsncmp(const struct bwstring *bws1, const struct bwstring *bws2, size_t offset, size_t len) { size_t cmp_len, len1, len2; int res = 0; len1 = bws1->len; len2 = bws2->len; if (len1 <= offset) { return ((len2 <= offset) ? 0 : -1); } else { if (len2 <= offset) return (+1); else { len1 -= offset; len2 -= offset; cmp_len = len1; if (len2 < cmp_len) cmp_len = len2; if (len < cmp_len) cmp_len = len; - if (MB_CUR_MAX == 1) { + if (mb_cur_max == 1) { const unsigned char *s1, *s2; s1 = bws1->data.cstr + offset; s2 = bws2->data.cstr + offset; res = memcmp(s1, s2, cmp_len); } else { const wchar_t *s1, *s2; s1 = bws1->data.wstr + offset; s2 = bws2->data.wstr + offset; res = memcmp(s1, s2, SIZEOF_WCHAR_STRING(cmp_len)); } } } if (res == 0) { if (len1 < cmp_len && len1 < len2) res = -1; else if (len2 < cmp_len && len2 < len1) res = +1; } return (res); } int bwscmp(const struct bwstring *bws1, const struct bwstring *bws2, size_t offset) { size_t len1, len2, cmp_len; int res; len1 = bws1->len; len2 = bws2->len; len1 -= offset; len2 -= offset; cmp_len = len1; if (len2 < cmp_len) cmp_len = len2; res = bwsncmp(bws1, bws2, offset, cmp_len); if (res == 0) { if( len1 < len2) res = -1; else if (len2 < len1) res = +1; } return (res); } int bws_iterator_cmp(bwstring_iterator iter1, bwstring_iterator iter2, size_t len) { wchar_t c1, c2; size_t i = 0; for (i = 0; i < len; ++i) { c1 = bws_get_iter_value(iter1); c2 = bws_get_iter_value(iter2); if (c1 != c2) return (c1 - c2); iter1 = bws_iterator_inc(iter1, 1); iter2 = bws_iterator_inc(iter2, 1); } return (0); } int bwscoll(const struct bwstring *bws1, const struct bwstring *bws2, size_t offset) { size_t len1, len2; len1 = bws1->len; len2 = bws2->len; if (len1 <= offset) return ((len2 <= offset) ? 0 : -1); else { if (len2 <= offset) return (+1); else { len1 -= offset; len2 -= offset; - if (MB_CUR_MAX == 1) { + if (mb_cur_max == 1) { const unsigned char *s1, *s2; s1 = bws1->data.cstr + offset; s2 = bws2->data.cstr + offset; if (byte_sort) { int res = 0; if (len1 > len2) { res = memcmp(s1, s2, len2); if (!res) res = +1; } else if (len1 < len2) { res = memcmp(s1, s2, len1); if (!res) res = -1; } else res = memcmp(s1, s2, len1); return (res); } else { int res = 0; size_t i, maxlen; i = 0; maxlen = len1; if (maxlen > len2) maxlen = len2; while (i < maxlen) { /* goto next non-zero part: */ while ((i < maxlen) && !s1[i] && !s2[i]) ++i; if (i >= maxlen) break; if (s1[i] == 0) { if (s2[i] == 0) /* NOTREACHED */ err(2, "bwscoll error 01"); else return (-1); } else if (s2[i] == 0) return (+1); res = strcoll((const char*)(s1 + i), (const char*)(s2 + i)); if (res) return (res); while ((i < maxlen) && s1[i] && s2[i]) ++i; if (i >= maxlen) break; if (s1[i] == 0) { if (s2[i] == 0) { ++i; continue; } else return (-1); } else if (s2[i] == 0) return (+1); else /* NOTREACHED */ err(2, "bwscoll error 02"); } if (len1 < len2) return (-1); else if (len1 > len2) return (+1); return (0); } } else { const wchar_t *s1, *s2; size_t i, maxlen; int res = 0; s1 = bws1->data.wstr + offset; s2 = bws2->data.wstr + offset; i = 0; maxlen = len1; if (maxlen > len2) maxlen = len2; while (i < maxlen) { /* goto next non-zero part: */ while ((i < maxlen) && !s1[i] && !s2[i]) ++i; if (i >= maxlen) break; if (s1[i] == 0) { if (s2[i] == 0) /* NOTREACHED */ err(2, "bwscoll error 1"); else return (-1); } else if (s2[i] == 0) return (+1); res = wide_str_coll(s1 + i, s2 + i); if (res) return (res); while ((i < maxlen) && s1[i] && s2[i]) ++i; if (i >= maxlen) break; if (s1[i] == 0) { if (s2[i] == 0) { ++i; continue; } else return (-1); } else if (s2[i] == 0) return (+1); else /* NOTREACHED */ err(2, "bwscoll error 2"); } if (len1 < len2) return (-1); else if (len1 > len2) return (+1); return (0); } } } } /* * Correction of the system API */ double bwstod(struct bwstring *s0, bool *empty) { double ret = 0; - if (MB_CUR_MAX == 1) { + if (mb_cur_max == 1) { unsigned char *end, *s; char *ep; s = s0->data.cstr; end = s + s0->len; ep = NULL; while (isblank(*s) && s < end) ++s; if (!isprint(*s)) { *empty = true; return (0); } ret = strtod((char*)s, &ep); if ((unsigned char*) ep == s) { *empty = true; return (0); } } else { wchar_t *end, *ep, *s; s = s0->data.wstr; end = s + s0->len; ep = NULL; while (iswblank(*s) && s < end) ++s; if (!iswprint(*s)) { *empty = true; return (0); } ret = wcstod(s, &ep); if (ep == s) { *empty = true; return (0); } } *empty = false; return (ret); } /* * A helper function for monthcoll. If a line matches * a month name, it returns (number of the month - 1), * while if there is no match, it just return -1. */ int bws_month_score(const struct bwstring *s0) { - if (MB_CUR_MAX == 1) { + if (mb_cur_max == 1) { const unsigned char *end, *s; s = s0->data.cstr; end = s + s0->len; while (isblank(*s) && s < end) ++s; for (int i = 11; i >= 0; --i) { if (cmonths[i] && (s == (unsigned char*)strstr((const char*)s, (char*)(cmonths[i])))) return (i); } } else { const wchar_t *end, *s; s = s0->data.wstr; end = s + s0->len; while (iswblank(*s) && s < end) ++s; for (int i = 11; i >= 0; --i) { if (wmonths[i] && (s == wcsstr(s, wmonths[i]))) return (i); } } return (-1); } /* * Rips out leading blanks (-b). */ struct bwstring * ignore_leading_blanks(struct bwstring *str) { - if (MB_CUR_MAX == 1) { + if (mb_cur_max == 1) { unsigned char *dst, *end, *src; src = str->data.cstr; dst = src; end = src + str->len; while (src < end && isblank(*src)) ++src; if (src != dst) { size_t newlen; newlen = BWSLEN(str) - (src - dst); while (src < end) { *dst = *src; ++dst; ++src; } bws_setlen(str, newlen); } } else { wchar_t *dst, *end, *src; src = str->data.wstr; dst = src; end = src + str->len; while (src < end && iswblank(*src)) ++src; if (src != dst) { size_t newlen = BWSLEN(str) - (src - dst); while (src < end) { *dst = *src; ++dst; ++src; } bws_setlen(str, newlen); } } return (str); } /* * Rips out nonprinting characters (-i). */ struct bwstring * ignore_nonprinting(struct bwstring *str) { size_t newlen = str->len; - if (MB_CUR_MAX == 1) { + if (mb_cur_max == 1) { unsigned char *dst, *end, *src; unsigned char c; src = str->data.cstr; dst = src; end = src + str->len; while (src < end) { c = *src; if (isprint(c)) { *dst = c; ++dst; ++src; } else { ++src; --newlen; } } } else { wchar_t *dst, *end, *src; wchar_t c; src = str->data.wstr; dst = src; end = src + str->len; while (src < end) { c = *src; if (iswprint(c)) { *dst = c; ++dst; ++src; } else { ++src; --newlen; } } } bws_setlen(str, newlen); return (str); } /* * Rips out any characters that are not alphanumeric characters * nor blanks (-d). */ struct bwstring * dictionary_order(struct bwstring *str) { size_t newlen = str->len; - if (MB_CUR_MAX == 1) { + if (mb_cur_max == 1) { unsigned char *dst, *end, *src; unsigned char c; src = str->data.cstr; dst = src; end = src + str->len; while (src < end) { c = *src; if (isalnum(c) || isblank(c)) { *dst = c; ++dst; ++src; } else { ++src; --newlen; } } } else { wchar_t *dst, *end, *src; wchar_t c; src = str->data.wstr; dst = src; end = src + str->len; while (src < end) { c = *src; if (iswalnum(c) || iswblank(c)) { *dst = c; ++dst; ++src; } else { ++src; --newlen; } } } bws_setlen(str, newlen); return (str); } /* * Converts string to lower case(-f). */ struct bwstring * ignore_case(struct bwstring *str) { - if (MB_CUR_MAX == 1) { + if (mb_cur_max == 1) { unsigned char *end, *s; s = str->data.cstr; end = s + str->len; while (s < end) { *s = toupper(*s); ++s; } } else { wchar_t *end, *s; s = str->data.wstr; end = s + str->len; while (s < end) { *s = towupper(*s); ++s; } } return (str); } void bws_disorder_warnx(struct bwstring *s, const char *fn, size_t pos) { - if (MB_CUR_MAX == 1) + if (mb_cur_max == 1) warnx("%s:%zu: disorder: %s", fn, pos + 1, s->data.cstr); else warnx("%s:%zu: disorder: %ls", fn, pos + 1, s->data.wstr); } diff --git a/usr.bin/sort/bwstring.h b/usr.bin/sort/bwstring.h index 9d62eaf37f62..b63bb97ab93f 100644 --- a/usr.bin/sort/bwstring.h +++ b/usr.bin/sort/bwstring.h @@ -1,144 +1,145 @@ /* $FreeBSD$ */ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (C) 2009 Gabor Kovesdan * Copyright (C) 2012 Oleg Moskalenko * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #if !defined(__BWSTRING_H__) #define __BWSTRING_H__ #include #include #include #include #include +#include "sort.h" #include "mem.h" extern bool byte_sort; /* wchar_t is of 4 bytes: */ #define SIZEOF_WCHAR_STRING(LEN) ((LEN)*sizeof(wchar_t)) /* * Binary "wide" string */ struct bwstring { size_t len; union { wchar_t wstr[0]; unsigned char cstr[0]; } data; }; struct reader_buffer { wchar_t *fgetwln_z_buffer; size_t fgetwln_z_buffer_size; }; typedef void *bwstring_iterator; #define BWSLEN(s) ((s)->len) struct bwstring *bwsalloc(size_t sz); size_t bwsrawlen(const struct bwstring *bws); const void* bwsrawdata(const struct bwstring *bws); void bws_setlen(struct bwstring *bws, size_t newlen); size_t bws_memsize(const struct bwstring *bws); double bwstod(struct bwstring *s0, bool *empty); int bws_month_score(const struct bwstring *s0); struct bwstring *ignore_leading_blanks(struct bwstring *str); struct bwstring *ignore_nonprinting(struct bwstring *str); struct bwstring *dictionary_order(struct bwstring *str); struct bwstring *ignore_case(struct bwstring *str); void bwsprintf(FILE*, struct bwstring*, const char *prefix, const char *suffix); void bws_disorder_warnx(struct bwstring *s, const char *fn, size_t pos); struct bwstring *bwsdup(const struct bwstring *s); struct bwstring *bwssbdup(const wchar_t *str, size_t size); struct bwstring *bwscsbdup(const unsigned char *str, size_t size); void bwsfree(const struct bwstring *s); size_t bwscpy(struct bwstring *dst, const struct bwstring *src); struct bwstring *bwsncpy(struct bwstring *dst, const struct bwstring *src, size_t size); struct bwstring *bwsnocpy(struct bwstring *dst, const struct bwstring *src, size_t offset, size_t size); int bwscmp(const struct bwstring *bws1, const struct bwstring *bws2, size_t offset); int bwsncmp(const struct bwstring *bws1, const struct bwstring *bws2, size_t offset, size_t len); int bwscoll(const struct bwstring *bws1, const struct bwstring *bws2, size_t offset); size_t bwsfwrite(struct bwstring *bws, FILE *f, bool zero_ended); struct bwstring *bwsfgetln(FILE *file, size_t *len, bool zero_ended, struct reader_buffer *rb); static inline bwstring_iterator bws_begin(struct bwstring *bws) { return (bwstring_iterator) (&(bws->data)); } static inline bwstring_iterator bws_end(struct bwstring *bws) { - return ((MB_CUR_MAX == 1) ? + return ((mb_cur_max == 1) ? (bwstring_iterator) (bws->data.cstr + bws->len) : (bwstring_iterator) (bws->data.wstr + bws->len)); } static inline bwstring_iterator bws_iterator_inc(bwstring_iterator iter, size_t pos) { - if (MB_CUR_MAX == 1) + if (mb_cur_max == 1) return ((unsigned char *) iter) + pos; else return ((wchar_t*) iter) + pos; } static inline wchar_t bws_get_iter_value(bwstring_iterator iter) { - if (MB_CUR_MAX == 1) + if (mb_cur_max == 1) return *((unsigned char *) iter); else return *((wchar_t*) iter); } int bws_iterator_cmp(bwstring_iterator iter1, bwstring_iterator iter2, size_t len); -#define BWS_GET(bws, pos) ((MB_CUR_MAX == 1) ? ((bws)->data.cstr[(pos)]) : (bws)->data.wstr[(pos)]) +#define BWS_GET(bws, pos) ((mb_cur_max == 1) ? ((bws)->data.cstr[(pos)]) : (bws)->data.wstr[(pos)]) void initialise_months(void); #endif /* __BWSTRING_H__ */ diff --git a/usr.bin/sort/radixsort.c b/usr.bin/sort/radixsort.c index 4993566aeb77..4c448fad69e9 100644 --- a/usr.bin/sort/radixsort.c +++ b/usr.bin/sort/radixsort.c @@ -1,728 +1,728 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (C) 2012 Oleg Moskalenko * Copyright (C) 2012 Gabor Kovesdan * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #if defined(SORT_THREADS) #include #include #endif #include #include #include #include #include #include "coll.h" #include "radixsort.h" #define DEFAULT_SORT_FUNC_RADIXSORT mergesort #define TINY_NODE(sl) ((sl)->tosort_num < 65) #define SMALL_NODE(sl) ((sl)->tosort_num < 5) /* are we sorting in reverse order ? */ static bool reverse_sort; /* sort sub-levels array size */ static const size_t slsz = 256 * sizeof(struct sort_level*); /* one sort level structure */ struct sort_level { struct sort_level **sublevels; struct sort_list_item **leaves; struct sort_list_item **sorted; struct sort_list_item **tosort; size_t leaves_num; size_t leaves_sz; size_t level; size_t real_sln; size_t start_position; size_t sln; size_t tosort_num; size_t tosort_sz; }; /* stack of sort levels ready to be sorted */ struct level_stack { struct level_stack *next; struct sort_level *sl; }; static struct level_stack *g_ls; #if defined(SORT_THREADS) /* stack guarding mutex */ static pthread_cond_t g_ls_cond; static pthread_mutex_t g_ls_mutex; /* counter: how many items are left */ static size_t sort_left; /* guarding mutex */ /* semaphore to count threads */ static sem_t mtsem; /* * Decrement items counter */ static inline void sort_left_dec(size_t n) { pthread_mutex_lock(&g_ls_mutex); sort_left -= n; if (sort_left == 0 && nthreads > 1) pthread_cond_broadcast(&g_ls_cond); pthread_mutex_unlock(&g_ls_mutex); } /* * Do we have something to sort ? * * This routine does not need to be locked. */ static inline bool have_sort_left(void) { bool ret; ret = (sort_left > 0); return (ret); } #else #define sort_left_dec(n) #endif /* SORT_THREADS */ static void _push_ls(struct level_stack *ls) { ls->next = g_ls; g_ls = ls; } /* * Push sort level to the stack */ static inline void push_ls(struct sort_level *sl) { struct level_stack *new_ls; new_ls = sort_malloc(sizeof(struct level_stack)); new_ls->sl = sl; #if defined(SORT_THREADS) if (nthreads > 1) { pthread_mutex_lock(&g_ls_mutex); _push_ls(new_ls); pthread_cond_signal(&g_ls_cond); pthread_mutex_unlock(&g_ls_mutex); } else #endif _push_ls(new_ls); } /* * Pop sort level from the stack (single-threaded style) */ static inline struct sort_level* pop_ls_st(void) { struct sort_level *sl; if (g_ls) { struct level_stack *saved_ls; sl = g_ls->sl; saved_ls = g_ls; g_ls = g_ls->next; sort_free(saved_ls); } else sl = NULL; return (sl); } #if defined(SORT_THREADS) /* * Pop sort level from the stack (multi-threaded style) */ static inline struct sort_level* pop_ls_mt(void) { struct level_stack *saved_ls; struct sort_level *sl; pthread_mutex_lock(&g_ls_mutex); for (;;) { if (g_ls) { sl = g_ls->sl; saved_ls = g_ls; g_ls = g_ls->next; break; } sl = NULL; saved_ls = NULL; if (have_sort_left() == 0) break; pthread_cond_wait(&g_ls_cond, &g_ls_mutex); } pthread_mutex_unlock(&g_ls_mutex); sort_free(saved_ls); return (sl); } #endif /* defined(SORT_THREADS) */ static void add_to_sublevel(struct sort_level *sl, struct sort_list_item *item, size_t indx) { struct sort_level *ssl; ssl = sl->sublevels[indx]; if (ssl == NULL) { ssl = sort_malloc(sizeof(struct sort_level)); memset(ssl, 0, sizeof(struct sort_level)); ssl->level = sl->level + 1; sl->sublevels[indx] = ssl; ++(sl->real_sln); } if (++(ssl->tosort_num) > ssl->tosort_sz) { ssl->tosort_sz = ssl->tosort_num + 128; ssl->tosort = sort_realloc(ssl->tosort, sizeof(struct sort_list_item*) * (ssl->tosort_sz)); } ssl->tosort[ssl->tosort_num - 1] = item; } static inline void add_leaf(struct sort_level *sl, struct sort_list_item *item) { if (++(sl->leaves_num) > sl->leaves_sz) { sl->leaves_sz = sl->leaves_num + 128; sl->leaves = sort_realloc(sl->leaves, (sizeof(struct sort_list_item*) * (sl->leaves_sz))); } sl->leaves[sl->leaves_num - 1] = item; } static inline int get_wc_index(struct sort_list_item *sli, size_t level) { - const size_t wcfact = (MB_CUR_MAX == 1) ? 1 : sizeof(wchar_t); + const size_t wcfact = (mb_cur_max == 1) ? 1 : sizeof(wchar_t); const struct key_value *kv; const struct bwstring *bws; kv = get_key_from_keys_array(&sli->ka, 0); bws = kv->k; if ((BWSLEN(bws) * wcfact > level)) { wchar_t res; /* * Sort wchar strings a byte at a time, rather than a single * byte from each wchar. */ res = (wchar_t)BWS_GET(bws, level / wcfact); /* Sort most-significant byte first. */ if (level % wcfact < wcfact - 1) res = (res >> (8 * (wcfact - 1 - (level % wcfact)))); return (res & 0xff); } return (-1); } static void place_item(struct sort_level *sl, size_t item) { struct sort_list_item *sli; int c; sli = sl->tosort[item]; c = get_wc_index(sli, sl->level); if (c == -1) add_leaf(sl, sli); else add_to_sublevel(sl, sli, c); } static void free_sort_level(struct sort_level *sl) { if (sl) { if (sl->leaves) sort_free(sl->leaves); if (sl->level > 0) sort_free(sl->tosort); if (sl->sublevels) { struct sort_level *slc; size_t sln; sln = sl->sln; for (size_t i = 0; i < sln; ++i) { slc = sl->sublevels[i]; if (slc) free_sort_level(slc); } sort_free(sl->sublevels); } sort_free(sl); } } static void run_sort_level_next(struct sort_level *sl) { - const size_t wcfact = (MB_CUR_MAX == 1) ? 1 : sizeof(wchar_t); + const size_t wcfact = (mb_cur_max == 1) ? 1 : sizeof(wchar_t); struct sort_level *slc; size_t i, sln, tosort_num; if (sl->sublevels) { sort_free(sl->sublevels); sl->sublevels = NULL; } switch (sl->tosort_num) { case 0: goto end; case (1): sl->sorted[sl->start_position] = sl->tosort[0]; sort_left_dec(1); goto end; case (2): /* * Radixsort only processes a single byte at a time. In wchar * mode, this can be a subset of the length of a character. * list_coll_offset() offset is in units of wchar, not bytes. * So to calculate the offset, we must divide by * sizeof(wchar_t) and round down to the index of the first * character this level references. */ if (list_coll_offset(&(sl->tosort[0]), &(sl->tosort[1]), sl->level / wcfact) > 0) { sl->sorted[sl->start_position++] = sl->tosort[1]; sl->sorted[sl->start_position] = sl->tosort[0]; } else { sl->sorted[sl->start_position++] = sl->tosort[0]; sl->sorted[sl->start_position] = sl->tosort[1]; } sort_left_dec(2); goto end; default: if (TINY_NODE(sl) || (sl->level > 15)) { listcoll_t func; /* * Collate comparison offset is in units of * character-width, so we must divide the level (bytes) * by operating character width (wchar_t or char). See * longer comment above. */ func = get_list_call_func(sl->level / wcfact); sl->leaves = sl->tosort; sl->leaves_num = sl->tosort_num; sl->leaves_sz = sl->leaves_num; sl->leaves = sort_realloc(sl->leaves, (sizeof(struct sort_list_item *) * (sl->leaves_sz))); sl->tosort = NULL; sl->tosort_num = 0; sl->tosort_sz = 0; sl->sln = 0; sl->real_sln = 0; if (sort_opts_vals.sflag) { if (mergesort(sl->leaves, sl->leaves_num, sizeof(struct sort_list_item *), (int(*)(const void *, const void *)) func) == -1) /* NOTREACHED */ err(2, "Radix sort error 3"); } else DEFAULT_SORT_FUNC_RADIXSORT(sl->leaves, sl->leaves_num, sizeof(struct sort_list_item *), (int(*)(const void *, const void *)) func); memcpy(sl->sorted + sl->start_position, sl->leaves, sl->leaves_num * sizeof(struct sort_list_item*)); sort_left_dec(sl->leaves_num); goto end; } else { sl->tosort_sz = sl->tosort_num; sl->tosort = sort_realloc(sl->tosort, sizeof(struct sort_list_item*) * (sl->tosort_sz)); } } sl->sln = 256; sl->sublevels = sort_malloc(slsz); memset(sl->sublevels, 0, slsz); sl->real_sln = 0; tosort_num = sl->tosort_num; for (i = 0; i < tosort_num; ++i) place_item(sl, i); sort_free(sl->tosort); sl->tosort = NULL; sl->tosort_num = 0; sl->tosort_sz = 0; if (sl->leaves_num > 1) { if (keys_num > 1) { if (sort_opts_vals.sflag) { mergesort(sl->leaves, sl->leaves_num, sizeof(struct sort_list_item *), (int(*)(const void *, const void *)) list_coll); } else { DEFAULT_SORT_FUNC_RADIXSORT(sl->leaves, sl->leaves_num, sizeof(struct sort_list_item *), (int(*)(const void *, const void *)) list_coll); } } else if (!sort_opts_vals.sflag && sort_opts_vals.complex_sort) { DEFAULT_SORT_FUNC_RADIXSORT(sl->leaves, sl->leaves_num, sizeof(struct sort_list_item *), (int(*)(const void *, const void *)) list_coll_by_str_only); } } sl->leaves_sz = sl->leaves_num; sl->leaves = sort_realloc(sl->leaves, (sizeof(struct sort_list_item *) * (sl->leaves_sz))); if (!reverse_sort) { memcpy(sl->sorted + sl->start_position, sl->leaves, sl->leaves_num * sizeof(struct sort_list_item*)); sl->start_position += sl->leaves_num; sort_left_dec(sl->leaves_num); sort_free(sl->leaves); sl->leaves = NULL; sl->leaves_num = 0; sl->leaves_sz = 0; sln = sl->sln; for (i = 0; i < sln; ++i) { slc = sl->sublevels[i]; if (slc) { slc->sorted = sl->sorted; slc->start_position = sl->start_position; sl->start_position += slc->tosort_num; if (SMALL_NODE(slc)) run_sort_level_next(slc); else push_ls(slc); sl->sublevels[i] = NULL; } } } else { size_t n; sln = sl->sln; for (i = 0; i < sln; ++i) { n = sln - i - 1; slc = sl->sublevels[n]; if (slc) { slc->sorted = sl->sorted; slc->start_position = sl->start_position; sl->start_position += slc->tosort_num; if (SMALL_NODE(slc)) run_sort_level_next(slc); else push_ls(slc); sl->sublevels[n] = NULL; } } memcpy(sl->sorted + sl->start_position, sl->leaves, sl->leaves_num * sizeof(struct sort_list_item*)); sort_left_dec(sl->leaves_num); } end: free_sort_level(sl); } /* * Single-threaded sort cycle */ static void run_sort_cycle_st(void) { struct sort_level *slc; for (;;) { slc = pop_ls_st(); if (slc == NULL) { break; } run_sort_level_next(slc); } } #if defined(SORT_THREADS) /* * Multi-threaded sort cycle */ static void run_sort_cycle_mt(void) { struct sort_level *slc; for (;;) { slc = pop_ls_mt(); if (slc == NULL) break; run_sort_level_next(slc); } } /* * Sort cycle thread (in multi-threaded mode) */ static void* sort_thread(void* arg) { run_sort_cycle_mt(); sem_post(&mtsem); return (arg); } #endif /* defined(SORT_THREADS) */ static void run_top_sort_level(struct sort_level *sl) { struct sort_level *slc; reverse_sort = sort_opts_vals.kflag ? keys[0].sm.rflag : default_sort_mods->rflag; sl->start_position = 0; sl->sln = 256; sl->sublevels = sort_malloc(slsz); memset(sl->sublevels, 0, slsz); for (size_t i = 0; i < sl->tosort_num; ++i) place_item(sl, i); if (sl->leaves_num > 1) { if (keys_num > 1) { if (sort_opts_vals.sflag) { mergesort(sl->leaves, sl->leaves_num, sizeof(struct sort_list_item *), (int(*)(const void *, const void *)) list_coll); } else { DEFAULT_SORT_FUNC_RADIXSORT(sl->leaves, sl->leaves_num, sizeof(struct sort_list_item *), (int(*)(const void *, const void *)) list_coll); } } else if (!sort_opts_vals.sflag && sort_opts_vals.complex_sort) { DEFAULT_SORT_FUNC_RADIXSORT(sl->leaves, sl->leaves_num, sizeof(struct sort_list_item *), (int(*)(const void *, const void *)) list_coll_by_str_only); } } if (!reverse_sort) { memcpy(sl->tosort + sl->start_position, sl->leaves, sl->leaves_num * sizeof(struct sort_list_item*)); sl->start_position += sl->leaves_num; sort_left_dec(sl->leaves_num); for (size_t i = 0; i < sl->sln; ++i) { slc = sl->sublevels[i]; if (slc) { slc->sorted = sl->tosort; slc->start_position = sl->start_position; sl->start_position += slc->tosort_num; push_ls(slc); sl->sublevels[i] = NULL; } } } else { size_t n; for (size_t i = 0; i < sl->sln; ++i) { n = sl->sln - i - 1; slc = sl->sublevels[n]; if (slc) { slc->sorted = sl->tosort; slc->start_position = sl->start_position; sl->start_position += slc->tosort_num; push_ls(slc); sl->sublevels[n] = NULL; } } memcpy(sl->tosort + sl->start_position, sl->leaves, sl->leaves_num * sizeof(struct sort_list_item*)); sort_left_dec(sl->leaves_num); } #if defined(SORT_THREADS) if (nthreads < 2) { #endif run_sort_cycle_st(); #if defined(SORT_THREADS) } else { size_t i; for(i = 0; i < nthreads; ++i) { pthread_attr_t attr; pthread_t pth; pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_DETACHED); for (;;) { int res = pthread_create(&pth, &attr, sort_thread, NULL); if (res >= 0) break; if (errno == EAGAIN) { pthread_yield(); continue; } err(2, NULL); } pthread_attr_destroy(&attr); } for (i = 0; i < nthreads; ++i) sem_wait(&mtsem); } #endif /* defined(SORT_THREADS) */ } static void run_sort(struct sort_list_item **base, size_t nmemb) { struct sort_level *sl; #if defined(SORT_THREADS) size_t nthreads_save = nthreads; if (nmemb < MT_SORT_THRESHOLD) nthreads = 1; if (nthreads > 1) { pthread_mutexattr_t mattr; pthread_mutexattr_init(&mattr); pthread_mutexattr_settype(&mattr, PTHREAD_MUTEX_ADAPTIVE_NP); pthread_mutex_init(&g_ls_mutex, &mattr); pthread_cond_init(&g_ls_cond, NULL); pthread_mutexattr_destroy(&mattr); sem_init(&mtsem, 0, 0); } #endif sl = sort_malloc(sizeof(struct sort_level)); memset(sl, 0, sizeof(struct sort_level)); sl->tosort = base; sl->tosort_num = nmemb; sl->tosort_sz = nmemb; #if defined(SORT_THREADS) sort_left = nmemb; #endif run_top_sort_level(sl); free_sort_level(sl); #if defined(SORT_THREADS) if (nthreads > 1) { sem_destroy(&mtsem); pthread_mutex_destroy(&g_ls_mutex); } nthreads = nthreads_save; #endif } void rxsort(struct sort_list_item **base, size_t nmemb) { run_sort(base, nmemb); } diff --git a/usr.bin/sort/sort.c b/usr.bin/sort/sort.c index d5c18552d914..0fbbd9284aad 100644 --- a/usr.bin/sort/sort.c +++ b/usr.bin/sort/sort.c @@ -1,1343 +1,1347 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (C) 2009 Gabor Kovesdan * Copyright (C) 2012 Oleg Moskalenko * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "coll.h" #include "file.h" #include "sort.h" #ifndef WITHOUT_NLS #include nl_catd catalog; #endif #define OPTIONS "bcCdfghik:Mmno:RrsS:t:T:uVz" static bool need_random; MD5_CTX md5_ctx; /* * Default messages to use when NLS is disabled or no catalogue * is found. */ const char *nlsstr[] = { "", /* 1*/"mutually exclusive flags", /* 2*/"extra argument not allowed with -c", /* 3*/"Unknown feature", /* 4*/"Wrong memory buffer specification", /* 5*/"0 field in key specs", /* 6*/"0 column in key specs", /* 7*/"Wrong file mode", /* 8*/"Cannot open file for reading", /* 9*/"Radix sort cannot be used with these sort options", /*10*/"The chosen sort method cannot be used with stable and/or unique sort", /*11*/"Invalid key position", /*12*/"Usage: %s [-bcCdfigMmnrsuz] [-kPOS1[,POS2] ... ] " "[+POS1 [-POS2]] [-S memsize] [-T tmpdir] [-t separator] " "[-o outfile] [--batch-size size] [--files0-from file] " "[--heapsort] [--mergesort] [--radixsort] [--qsort] " "[--mmap] " #if defined(SORT_THREADS) "[--parallel thread_no] " #endif "[--human-numeric-sort] " "[--version-sort] [--random-sort [--random-source file]] " "[--compress-program program] [file ...]\n" }; struct sort_opts sort_opts_vals; bool debug_sort; bool need_hint; +size_t mb_cur_max; + #if defined(SORT_THREADS) unsigned int ncpu = 1; size_t nthreads = 1; #endif static bool gnusort_numeric_compatibility; static struct sort_mods default_sort_mods_object; struct sort_mods * const default_sort_mods = &default_sort_mods_object; static bool print_symbols_on_debug; /* * Arguments from file (when file0-from option is used: */ static size_t argc_from_file0 = (size_t)-1; static char **argv_from_file0; /* * Placeholder symbols for options which have no single-character equivalent */ enum { SORT_OPT = CHAR_MAX + 1, HELP_OPT, FF_OPT, BS_OPT, VERSION_OPT, DEBUG_OPT, #if defined(SORT_THREADS) PARALLEL_OPT, #endif RANDOMSOURCE_OPT, COMPRESSPROGRAM_OPT, QSORT_OPT, MERGESORT_OPT, HEAPSORT_OPT, RADIXSORT_OPT, MMAP_OPT }; #define NUMBER_OF_MUTUALLY_EXCLUSIVE_FLAGS 6 static const char mutually_exclusive_flags[NUMBER_OF_MUTUALLY_EXCLUSIVE_FLAGS] = { 'M', 'n', 'g', 'R', 'h', 'V' }; static struct option long_options[] = { { "batch-size", required_argument, NULL, BS_OPT }, { "buffer-size", required_argument, NULL, 'S' }, { "check", optional_argument, NULL, 'c' }, { "check=silent|quiet", optional_argument, NULL, 'C' }, { "compress-program", required_argument, NULL, COMPRESSPROGRAM_OPT }, { "debug", no_argument, NULL, DEBUG_OPT }, { "dictionary-order", no_argument, NULL, 'd' }, { "field-separator", required_argument, NULL, 't' }, { "files0-from", required_argument, NULL, FF_OPT }, { "general-numeric-sort", no_argument, NULL, 'g' }, { "heapsort", no_argument, NULL, HEAPSORT_OPT }, { "help",no_argument, NULL, HELP_OPT }, { "human-numeric-sort", no_argument, NULL, 'h' }, { "ignore-leading-blanks", no_argument, NULL, 'b' }, { "ignore-case", no_argument, NULL, 'f' }, { "ignore-nonprinting", no_argument, NULL, 'i' }, { "key", required_argument, NULL, 'k' }, { "merge", no_argument, NULL, 'm' }, { "mergesort", no_argument, NULL, MERGESORT_OPT }, { "mmap", no_argument, NULL, MMAP_OPT }, { "month-sort", no_argument, NULL, 'M' }, { "numeric-sort", no_argument, NULL, 'n' }, { "output", required_argument, NULL, 'o' }, #if defined(SORT_THREADS) { "parallel", required_argument, NULL, PARALLEL_OPT }, #endif { "qsort", no_argument, NULL, QSORT_OPT }, { "radixsort", no_argument, NULL, RADIXSORT_OPT }, { "random-sort", no_argument, NULL, 'R' }, { "random-source", required_argument, NULL, RANDOMSOURCE_OPT }, { "reverse", no_argument, NULL, 'r' }, { "sort", required_argument, NULL, SORT_OPT }, { "stable", no_argument, NULL, 's' }, { "temporary-directory",required_argument, NULL, 'T' }, { "unique", no_argument, NULL, 'u' }, { "version", no_argument, NULL, VERSION_OPT }, { "version-sort",no_argument, NULL, 'V' }, { "zero-terminated", no_argument, NULL, 'z' }, { NULL, no_argument, NULL, 0 } }; void fix_obsolete_keys(int *argc, char **argv); /* * Check where sort modifier is present */ static bool sort_modifier_empty(struct sort_mods *sm) { if (sm == NULL) return (true); return (!(sm->Mflag || sm->Vflag || sm->nflag || sm->gflag || sm->rflag || sm->Rflag || sm->hflag || sm->dflag || sm->fflag)); } /* * Print out usage text. */ static void usage(bool opt_err) { FILE *out; out = opt_err ? stderr : stdout; fprintf(out, getstr(12), getprogname()); if (opt_err) exit(2); exit(0); } /* * Read input file names from a file (file0-from option). */ static void read_fns_from_file0(const char *fn) { FILE *f; char *line = NULL; size_t linesize = 0; ssize_t linelen; if (fn == NULL) return; f = fopen(fn, "r"); if (f == NULL) err(2, "%s", fn); while ((linelen = getdelim(&line, &linesize, '\0', f)) != -1) { if (*line != '\0') { if (argc_from_file0 == (size_t) - 1) argc_from_file0 = 0; ++argc_from_file0; argv_from_file0 = sort_realloc(argv_from_file0, argc_from_file0 * sizeof(char *)); if (argv_from_file0 == NULL) err(2, NULL); argv_from_file0[argc_from_file0 - 1] = line; } else { free(line); } line = NULL; linesize = 0; } if (ferror(f)) err(2, "%s: getdelim", fn); closefile(f, fn); } /* * Check how much RAM is available for the sort. */ static void set_hw_params(void) { long pages, psize; #if defined(SORT_THREADS) ncpu = 1; #endif pages = sysconf(_SC_PHYS_PAGES); if (pages < 1) { perror("sysconf pages"); pages = 1; } psize = sysconf(_SC_PAGESIZE); if (psize < 1) { perror("sysconf psize"); psize = 4096; } #if defined(SORT_THREADS) ncpu = (unsigned int)sysconf(_SC_NPROCESSORS_ONLN); if (ncpu < 1) ncpu = 1; else if(ncpu > 32) ncpu = 32; nthreads = ncpu; #endif free_memory = (unsigned long long) pages * (unsigned long long) psize; available_free_memory = free_memory / 2; if (available_free_memory < 1024) available_free_memory = 1024; } /* * Convert "plain" symbol to wide symbol, with default value. */ static void conv_mbtowc(wchar_t *wc, const char *c, const wchar_t def) { if (wc && c) { int res; - res = mbtowc(wc, c, MB_CUR_MAX); + res = mbtowc(wc, c, mb_cur_max); if (res < 1) *wc = def; } } /* * Set current locale symbols. */ static void set_locale(void) { struct lconv *lc; const char *locale; setlocale(LC_ALL, ""); + mb_cur_max = MB_CUR_MAX; + lc = localeconv(); if (lc) { /* obtain LC_NUMERIC info */ /* Convert to wide char form */ conv_mbtowc(&symbol_decimal_point, lc->decimal_point, symbol_decimal_point); conv_mbtowc(&symbol_thousands_sep, lc->thousands_sep, symbol_thousands_sep); conv_mbtowc(&symbol_positive_sign, lc->positive_sign, symbol_positive_sign); conv_mbtowc(&symbol_negative_sign, lc->negative_sign, symbol_negative_sign); } if (getenv("GNUSORT_NUMERIC_COMPATIBILITY")) gnusort_numeric_compatibility = true; locale = setlocale(LC_COLLATE, NULL); if (locale) { char *tmpl; const char *cclocale; tmpl = sort_strdup(locale); cclocale = setlocale(LC_COLLATE, "C"); if (cclocale && !strcmp(cclocale, tmpl)) byte_sort = true; else { const char *pclocale; pclocale = setlocale(LC_COLLATE, "POSIX"); if (pclocale && !strcmp(pclocale, tmpl)) byte_sort = true; } setlocale(LC_COLLATE, tmpl); sort_free(tmpl); } } /* * Set directory temporary files. */ static void set_tmpdir(void) { char *td; td = getenv("TMPDIR"); if (td != NULL) tmpdir = sort_strdup(td); } /* * Parse -S option. */ static unsigned long long parse_memory_buffer_value(const char *value) { if (value == NULL) return (available_free_memory); else { char *endptr; unsigned long long membuf; endptr = NULL; errno = 0; membuf = strtoll(value, &endptr, 10); if (errno != 0) { warn("%s",getstr(4)); membuf = available_free_memory; } else { switch (*endptr){ case 'Y': membuf *= 1024; /* FALLTHROUGH */ case 'Z': membuf *= 1024; /* FALLTHROUGH */ case 'E': membuf *= 1024; /* FALLTHROUGH */ case 'P': membuf *= 1024; /* FALLTHROUGH */ case 'T': membuf *= 1024; /* FALLTHROUGH */ case 'G': membuf *= 1024; /* FALLTHROUGH */ case 'M': membuf *= 1024; /* FALLTHROUGH */ case '\0': case 'K': membuf *= 1024; /* FALLTHROUGH */ case 'b': break; case '%': membuf = (available_free_memory * membuf) / 100; break; default: warnc(EINVAL, "%s", optarg); membuf = available_free_memory; } } return (membuf); } } /* * Signal handler that clears the temporary files. */ static void sig_handler(int sig __unused, siginfo_t *siginfo __unused, void *context __unused) { clear_tmp_files(); exit(-1); } /* * Set signal handler on panic signals. */ static void set_signal_handler(void) { struct sigaction sa; memset(&sa, 0, sizeof(sa)); sa.sa_sigaction = &sig_handler; sa.sa_flags = SA_SIGINFO; if (sigaction(SIGTERM, &sa, NULL) < 0) { perror("sigaction"); return; } if (sigaction(SIGHUP, &sa, NULL) < 0) { perror("sigaction"); return; } if (sigaction(SIGINT, &sa, NULL) < 0) { perror("sigaction"); return; } if (sigaction(SIGQUIT, &sa, NULL) < 0) { perror("sigaction"); return; } if (sigaction(SIGABRT, &sa, NULL) < 0) { perror("sigaction"); return; } if (sigaction(SIGBUS, &sa, NULL) < 0) { perror("sigaction"); return; } if (sigaction(SIGSEGV, &sa, NULL) < 0) { perror("sigaction"); return; } if (sigaction(SIGUSR1, &sa, NULL) < 0) { perror("sigaction"); return; } if (sigaction(SIGUSR2, &sa, NULL) < 0) { perror("sigaction"); return; } } /* * Print "unknown" message and exit with status 2. */ static void unknown(const char *what) { errx(2, "%s: %s", getstr(3), what); } /* * Check whether contradictory input options are used. */ static void check_mutually_exclusive_flags(char c, bool *mef_flags) { int fo_index, mec; bool found_others, found_this; found_others = found_this = false; fo_index = 0; for (int i = 0; i < NUMBER_OF_MUTUALLY_EXCLUSIVE_FLAGS; i++) { mec = mutually_exclusive_flags[i]; if (mec != c) { if (mef_flags[i]) { if (found_this) errx(1, "%c:%c: %s", c, mec, getstr(1)); found_others = true; fo_index = i; } } else { if (found_others) errx(1, "%c:%c: %s", c, mutually_exclusive_flags[fo_index], getstr(1)); mef_flags[i] = true; found_this = true; } } } /* * Initialise sort opts data. */ static void set_sort_opts(void) { memset(&default_sort_mods_object, 0, sizeof(default_sort_mods_object)); memset(&sort_opts_vals, 0, sizeof(sort_opts_vals)); default_sort_mods_object.func = get_sort_func(&default_sort_mods_object); } /* * Set a sort modifier on a sort modifiers object. */ static bool set_sort_modifier(struct sort_mods *sm, int c) { if (sm == NULL) return (true); switch (c){ case 'b': sm->bflag = true; break; case 'd': sm->dflag = true; break; case 'f': sm->fflag = true; break; case 'g': sm->gflag = true; need_hint = true; break; case 'i': sm->iflag = true; break; case 'R': sm->Rflag = true; need_hint = true; need_random = true; break; case 'M': initialise_months(); sm->Mflag = true; need_hint = true; break; case 'n': sm->nflag = true; need_hint = true; print_symbols_on_debug = true; break; case 'r': sm->rflag = true; break; case 'V': sm->Vflag = true; break; case 'h': sm->hflag = true; need_hint = true; print_symbols_on_debug = true; break; default: return (false); } sort_opts_vals.complex_sort = true; sm->func = get_sort_func(sm); return (true); } /* * Parse POS in -k option. */ static int parse_pos(const char *s, struct key_specs *ks, bool *mef_flags, bool second) { regmatch_t pmatch[4]; regex_t re; char *c, *f; const char *sregexp = "^([0-9]+)(\\.[0-9]+)?([bdfirMngRhV]+)?$"; size_t len, nmatch; int ret; ret = -1; nmatch = 4; c = f = NULL; if (regcomp(&re, sregexp, REG_EXTENDED) != 0) return (-1); if (regexec(&re, s, nmatch, pmatch, 0) != 0) goto end; if (pmatch[0].rm_eo <= pmatch[0].rm_so) goto end; if (pmatch[1].rm_eo <= pmatch[1].rm_so) goto end; len = pmatch[1].rm_eo - pmatch[1].rm_so; f = sort_malloc((len + 1) * sizeof(char)); strncpy(f, s + pmatch[1].rm_so, len); f[len] = '\0'; if (second) { errno = 0; ks->f2 = (size_t) strtoul(f, NULL, 10); if (errno != 0) err(2, "-k"); if (ks->f2 == 0) { warn("%s",getstr(5)); goto end; } } else { errno = 0; ks->f1 = (size_t) strtoul(f, NULL, 10); if (errno != 0) err(2, "-k"); if (ks->f1 == 0) { warn("%s",getstr(5)); goto end; } } if (pmatch[2].rm_eo > pmatch[2].rm_so) { len = pmatch[2].rm_eo - pmatch[2].rm_so - 1; c = sort_malloc((len + 1) * sizeof(char)); strncpy(c, s + pmatch[2].rm_so + 1, len); c[len] = '\0'; if (second) { errno = 0; ks->c2 = (size_t) strtoul(c, NULL, 10); if (errno != 0) err(2, "-k"); } else { errno = 0; ks->c1 = (size_t) strtoul(c, NULL, 10); if (errno != 0) err(2, "-k"); if (ks->c1 == 0) { warn("%s",getstr(6)); goto end; } } } else { if (second) ks->c2 = 0; else ks->c1 = 1; } if (pmatch[3].rm_eo > pmatch[3].rm_so) { regoff_t i = 0; for (i = pmatch[3].rm_so; i < pmatch[3].rm_eo; i++) { check_mutually_exclusive_flags(s[i], mef_flags); if (s[i] == 'b') { if (second) ks->pos2b = true; else ks->pos1b = true; } else if (!set_sort_modifier(&(ks->sm), s[i])) goto end; } } ret = 0; end: if (c) sort_free(c); if (f) sort_free(f); regfree(&re); return (ret); } /* * Parse -k option value. */ static int parse_k(const char *s, struct key_specs *ks) { int ret = -1; bool mef_flags[NUMBER_OF_MUTUALLY_EXCLUSIVE_FLAGS] = { false, false, false, false, false, false }; if (s && *s) { char *sptr; sptr = strchr(s, ','); if (sptr) { size_t size1; char *pos1, *pos2; size1 = sptr - s; if (size1 < 1) return (-1); pos1 = sort_malloc((size1 + 1) * sizeof(char)); strncpy(pos1, s, size1); pos1[size1] = '\0'; ret = parse_pos(pos1, ks, mef_flags, false); sort_free(pos1); if (ret < 0) return (ret); pos2 = sort_strdup(sptr + 1); ret = parse_pos(pos2, ks, mef_flags, true); sort_free(pos2); } else ret = parse_pos(s, ks, mef_flags, false); } return (ret); } /* * Parse POS in +POS -POS option. */ static int parse_pos_obs(const char *s, int *nf, int *nc, char* sopts) { regex_t re; regmatch_t pmatch[4]; char *c, *f; const char *sregexp = "^([0-9]+)(\\.[0-9]+)?([A-Za-z]+)?$"; int ret; size_t len, nmatch; ret = -1; nmatch = 4; c = f = NULL; *nc = *nf = 0; if (regcomp(&re, sregexp, REG_EXTENDED) != 0) return (-1); if (regexec(&re, s, nmatch, pmatch, 0) != 0) goto end; if (pmatch[0].rm_eo <= pmatch[0].rm_so) goto end; if (pmatch[1].rm_eo <= pmatch[1].rm_so) goto end; len = pmatch[1].rm_eo - pmatch[1].rm_so; f = sort_malloc((len + 1) * sizeof(char)); strncpy(f, s + pmatch[1].rm_so, len); f[len] = '\0'; errno = 0; *nf = (size_t) strtoul(f, NULL, 10); if (errno != 0) errx(2, "%s", getstr(11)); if (pmatch[2].rm_eo > pmatch[2].rm_so) { len = pmatch[2].rm_eo - pmatch[2].rm_so - 1; c = sort_malloc((len + 1) * sizeof(char)); strncpy(c, s + pmatch[2].rm_so + 1, len); c[len] = '\0'; errno = 0; *nc = (size_t) strtoul(c, NULL, 10); if (errno != 0) errx(2, "%s", getstr(11)); } if (pmatch[3].rm_eo > pmatch[3].rm_so) { len = pmatch[3].rm_eo - pmatch[3].rm_so; strncpy(sopts, s + pmatch[3].rm_so, len); sopts[len] = '\0'; } ret = 0; end: if (c) sort_free(c); if (f) sort_free(f); regfree(&re); return (ret); } /* * "Translate" obsolete +POS1 -POS2 syntax into new -kPOS1,POS2 syntax */ void fix_obsolete_keys(int *argc, char **argv) { char sopt[129]; for (int i = 1; i < *argc; i++) { char *arg1; arg1 = argv[i]; if (strcmp(arg1, "--") == 0) { /* Following arguments are treated as filenames. */ break; } if (strlen(arg1) > 1 && arg1[0] == '+') { int c1, f1; char sopts1[128]; sopts1[0] = 0; c1 = f1 = 0; if (parse_pos_obs(arg1 + 1, &f1, &c1, sopts1) < 0) continue; else { f1 += 1; c1 += 1; if (i + 1 < *argc) { char *arg2 = argv[i + 1]; if (strlen(arg2) > 1 && arg2[0] == '-') { int c2, f2; char sopts2[128]; sopts2[0] = 0; c2 = f2 = 0; if (parse_pos_obs(arg2 + 1, &f2, &c2, sopts2) >= 0) { if (c2 > 0) f2 += 1; sprintf(sopt, "-k%d.%d%s,%d.%d%s", f1, c1, sopts1, f2, c2, sopts2); argv[i] = sort_strdup(sopt); for (int j = i + 1; j + 1 < *argc; j++) argv[j] = argv[j + 1]; *argc -= 1; continue; } } } sprintf(sopt, "-k%d.%d%s", f1, c1, sopts1); argv[i] = sort_strdup(sopt); } } } } /* * Seed random sort */ static void get_random_seed(const char *random_source) { char randseed[32]; struct stat fsb, rsb; ssize_t rd; int rsfd; rsfd = -1; rd = sizeof(randseed); if (random_source == NULL) { if (getentropy(randseed, sizeof(randseed)) < 0) err(EX_SOFTWARE, "getentropy"); goto out; } rsfd = open(random_source, O_RDONLY | O_CLOEXEC); if (rsfd < 0) err(EX_NOINPUT, "open: %s", random_source); if (fstat(rsfd, &fsb) != 0) err(EX_SOFTWARE, "fstat"); if (!S_ISREG(fsb.st_mode) && !S_ISCHR(fsb.st_mode)) err(EX_USAGE, "random seed isn't a regular file or /dev/random"); /* * Regular files: read up to maximum seed size and explicitly * reject longer files. */ if (S_ISREG(fsb.st_mode)) { if (fsb.st_size > (off_t)sizeof(randseed)) errx(EX_USAGE, "random seed is too large (%jd >" " %zu)!", (intmax_t)fsb.st_size, sizeof(randseed)); else if (fsb.st_size < 1) errx(EX_USAGE, "random seed is too small (" "0 bytes)"); memset(randseed, 0, sizeof(randseed)); rd = read(rsfd, randseed, fsb.st_size); if (rd < 0) err(EX_SOFTWARE, "reading random seed file %s", random_source); if (rd < (ssize_t)fsb.st_size) errx(EX_SOFTWARE, "short read from %s", random_source); } else if (S_ISCHR(fsb.st_mode)) { if (stat("/dev/random", &rsb) < 0) err(EX_SOFTWARE, "stat"); if (fsb.st_dev != rsb.st_dev || fsb.st_ino != rsb.st_ino) errx(EX_USAGE, "random seed is a character " "device other than /dev/random"); if (getentropy(randseed, sizeof(randseed)) < 0) err(EX_SOFTWARE, "getentropy"); } out: if (rsfd >= 0) close(rsfd); MD5Init(&md5_ctx); MD5Update(&md5_ctx, randseed, rd); } /* * Main function. */ int main(int argc, char **argv) { char *outfile, *real_outfile; char *random_source = NULL; int c, result; bool mef_flags[NUMBER_OF_MUTUALLY_EXCLUSIVE_FLAGS] = { false, false, false, false, false, false }; result = 0; outfile = sort_strdup("-"); real_outfile = NULL; struct sort_mods *sm = &default_sort_mods_object; init_tmp_files(); set_signal_handler(); set_hw_params(); set_locale(); set_tmpdir(); set_sort_opts(); fix_obsolete_keys(&argc, argv); while (((c = getopt_long(argc, argv, OPTIONS, long_options, NULL)) != -1)) { check_mutually_exclusive_flags(c, mef_flags); if (!set_sort_modifier(sm, c)) { switch (c) { case 'c': sort_opts_vals.cflag = true; if (optarg) { if (!strcmp(optarg, "diagnose-first")) ; else if (!strcmp(optarg, "silent") || !strcmp(optarg, "quiet")) sort_opts_vals.csilentflag = true; else if (*optarg) unknown(optarg); } break; case 'C': sort_opts_vals.cflag = true; sort_opts_vals.csilentflag = true; break; case 'k': { sort_opts_vals.complex_sort = true; sort_opts_vals.kflag = true; keys_num++; keys = sort_realloc(keys, keys_num * sizeof(struct key_specs)); memset(&(keys[keys_num - 1]), 0, sizeof(struct key_specs)); if (parse_k(optarg, &(keys[keys_num - 1])) < 0) { errc(2, EINVAL, "-k %s", optarg); } break; } case 'm': sort_opts_vals.mflag = true; break; case 'o': outfile = sort_realloc(outfile, (strlen(optarg) + 1)); strcpy(outfile, optarg); break; case 's': sort_opts_vals.sflag = true; break; case 'S': available_free_memory = parse_memory_buffer_value(optarg); break; case 'T': tmpdir = sort_strdup(optarg); break; case 't': while (strlen(optarg) > 1) { if (optarg[0] != '\\') { errc(2, EINVAL, "%s", optarg); } optarg += 1; if (*optarg == '0') { *optarg = 0; break; } } sort_opts_vals.tflag = true; sort_opts_vals.field_sep = btowc(optarg[0]); if (sort_opts_vals.field_sep == WEOF) { errno = EINVAL; err(2, NULL); } if (!gnusort_numeric_compatibility) { if (symbol_decimal_point == sort_opts_vals.field_sep) symbol_decimal_point = WEOF; if (symbol_thousands_sep == sort_opts_vals.field_sep) symbol_thousands_sep = WEOF; if (symbol_negative_sign == sort_opts_vals.field_sep) symbol_negative_sign = WEOF; if (symbol_positive_sign == sort_opts_vals.field_sep) symbol_positive_sign = WEOF; } break; case 'u': sort_opts_vals.uflag = true; /* stable sort for the correct unique val */ sort_opts_vals.sflag = true; break; case 'z': sort_opts_vals.zflag = true; break; case SORT_OPT: if (optarg) { if (!strcmp(optarg, "general-numeric")) set_sort_modifier(sm, 'g'); else if (!strcmp(optarg, "human-numeric")) set_sort_modifier(sm, 'h'); else if (!strcmp(optarg, "numeric")) set_sort_modifier(sm, 'n'); else if (!strcmp(optarg, "month")) set_sort_modifier(sm, 'M'); else if (!strcmp(optarg, "random")) set_sort_modifier(sm, 'R'); else unknown(optarg); } break; #if defined(SORT_THREADS) case PARALLEL_OPT: nthreads = (size_t)(atoi(optarg)); if (nthreads < 1) nthreads = 1; if (nthreads > 1024) nthreads = 1024; break; #endif case QSORT_OPT: sort_opts_vals.sort_method = SORT_QSORT; break; case MERGESORT_OPT: sort_opts_vals.sort_method = SORT_MERGESORT; break; case MMAP_OPT: use_mmap = true; break; case HEAPSORT_OPT: sort_opts_vals.sort_method = SORT_HEAPSORT; break; case RADIXSORT_OPT: sort_opts_vals.sort_method = SORT_RADIXSORT; break; case RANDOMSOURCE_OPT: random_source = strdup(optarg); break; case COMPRESSPROGRAM_OPT: compress_program = strdup(optarg); break; case FF_OPT: read_fns_from_file0(optarg); break; case BS_OPT: { errno = 0; long mof = strtol(optarg, NULL, 10); if (errno != 0) err(2, "--batch-size"); if (mof >= 2) max_open_files = (size_t) mof + 1; } break; case VERSION_OPT: printf("%s\n", VERSION); exit(EXIT_SUCCESS); /* NOTREACHED */ break; case DEBUG_OPT: debug_sort = true; break; case HELP_OPT: usage(false); /* NOTREACHED */ break; default: usage(true); /* NOTREACHED */ } } } argc -= optind; argv += optind; if (argv_from_file0) { argc = argc_from_file0; argv = argv_from_file0; } #ifndef WITHOUT_NLS catalog = catopen("sort", NL_CAT_LOCALE); #endif if (sort_opts_vals.cflag && sort_opts_vals.mflag) errx(1, "%c:%c: %s", 'm', 'c', getstr(1)); #ifndef WITHOUT_NLS catclose(catalog); #endif if (keys_num == 0) { keys_num = 1; keys = sort_realloc(keys, sizeof(struct key_specs)); memset(&(keys[0]), 0, sizeof(struct key_specs)); keys[0].c1 = 1; keys[0].pos1b = default_sort_mods->bflag; keys[0].pos2b = default_sort_mods->bflag; memcpy(&(keys[0].sm), default_sort_mods, sizeof(struct sort_mods)); } for (size_t i = 0; i < keys_num; i++) { struct key_specs *ks; ks = &(keys[i]); if (sort_modifier_empty(&(ks->sm)) && !(ks->pos1b) && !(ks->pos2b)) { ks->pos1b = sm->bflag; ks->pos2b = sm->bflag; memcpy(&(ks->sm), sm, sizeof(struct sort_mods)); } ks->sm.func = get_sort_func(&(ks->sm)); } if (debug_sort) { printf("Memory to be used for sorting: %llu\n",available_free_memory); #if defined(SORT_THREADS) printf("Number of CPUs: %d\n",(int)ncpu); nthreads = 1; #endif printf("Using collate rules of %s locale\n", setlocale(LC_COLLATE, NULL)); if (byte_sort) printf("Byte sort is used\n"); if (print_symbols_on_debug) { printf("Decimal Point: <%lc>\n", symbol_decimal_point); if (symbol_thousands_sep) printf("Thousands separator: <%lc>\n", symbol_thousands_sep); printf("Positive sign: <%lc>\n", symbol_positive_sign); printf("Negative sign: <%lc>\n", symbol_negative_sign); } } if (need_random) get_random_seed(random_source); /* Case when the outfile equals one of the input files: */ if (strcmp(outfile, "-")) { for(int i = 0; i < argc; ++i) { if (strcmp(argv[i], outfile) == 0) { real_outfile = sort_strdup(outfile); for(;;) { char* tmp = sort_malloc(strlen(outfile) + strlen(".tmp") + 1); strcpy(tmp, outfile); strcpy(tmp + strlen(tmp), ".tmp"); sort_free(outfile); outfile = tmp; if (access(outfile, F_OK) < 0) break; } tmp_file_atexit(outfile); } } } #if defined(SORT_THREADS) if ((argc < 1) || (strcmp(outfile, "-") == 0) || (*outfile == 0)) nthreads = 1; #endif if (!sort_opts_vals.cflag && !sort_opts_vals.mflag) { struct file_list fl; struct sort_list list; sort_list_init(&list); file_list_init(&fl, true); if (argc < 1) procfile("-", &list, &fl); else { while (argc > 0) { procfile(*argv, &list, &fl); --argc; ++argv; } } if (fl.count < 1) sort_list_to_file(&list, outfile); else { if (list.count > 0) { char *flast = new_tmp_file_name(); sort_list_to_file(&list, flast); file_list_add(&fl, flast, false); } merge_files(&fl, outfile); } file_list_clean(&fl); /* * We are about to exit the program, so we can ignore * the clean-up for speed * * sort_list_clean(&list); */ } else if (sort_opts_vals.cflag) { result = (argc == 0) ? (check("-")) : (check(*argv)); } else if (sort_opts_vals.mflag) { struct file_list fl; file_list_init(&fl, false); /* No file arguments remaining means "read from stdin." */ if (argc == 0) file_list_add(&fl, "-", true); else file_list_populate(&fl, argc, argv, true); merge_files(&fl, outfile); file_list_clean(&fl); } if (real_outfile) { unlink(real_outfile); if (rename(outfile, real_outfile) < 0) err(2, NULL); sort_free(real_outfile); } sort_free(outfile); return (result); } diff --git a/usr.bin/sort/sort.h b/usr.bin/sort/sort.h index b472fd16c098..96b36a65f4e4 100644 --- a/usr.bin/sort/sort.h +++ b/usr.bin/sort/sort.h @@ -1,130 +1,136 @@ /* $FreeBSD$ */ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (C) 2009 Gabor Kovesdan * Copyright (C) 2012 Oleg Moskalenko * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #if !defined(__BSD_SORT_H__) #define __BSD_SORT_H__ #include #include #include #include #include #include #include #define VERSION "2.3-FreeBSD" #ifdef WITHOUT_NLS #define getstr(n) nlsstr[n] #else #include extern nl_catd catalog; #define getstr(n) catgets(catalog, 1, n, nlsstr[n]) #endif extern const char *nlsstr[]; #if defined(SORT_THREADS) #define MT_SORT_THRESHOLD (10000) extern unsigned int ncpu; extern size_t nthreads; #endif /* * If true, we output some debug information. */ extern bool debug_sort; /* * MD5 context for random hash function */ extern MD5_CTX md5_ctx; /* * sort.c */ /* * This structure holds main sort options which are NOT affecting the sort ordering. */ struct sort_opts { wint_t field_sep; int sort_method; bool cflag; bool csilentflag; bool kflag; bool mflag; bool sflag; bool uflag; bool zflag; bool tflag; bool complex_sort; }; /* * Key value structure forward declaration */ struct key_value; /* * Cmp function */ typedef int (*cmpcoll_t)(struct key_value *kv1, struct key_value *kv2, size_t offset); /* * This structure holds "sort modifiers" - options which are affecting the sort ordering. */ struct sort_mods { cmpcoll_t func; bool bflag; bool dflag; bool fflag; bool gflag; bool iflag; bool Mflag; bool nflag; bool rflag; bool Rflag; bool Vflag; bool hflag; }; extern bool need_hint; extern struct sort_opts sort_opts_vals; extern struct sort_mods * const default_sort_mods; +/* + * Cached value of MB_CUR_MAX. Because MB_CUR_MAX is used often throughout the program, + * this avoids frequent calls to __mb_cur_max. + */ +extern size_t mb_cur_max; + #endif /* __BSD_SORT_H__ */