diff --git a/usr.bin/sort/bwstring.c b/usr.bin/sort/bwstring.c index 2f2737e94314..52173002986f 100644 --- a/usr.bin/sort/bwstring.c +++ b/usr.bin/sort/bwstring.c @@ -1,1147 +1,1038 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (C) 2009 Gabor Kovesdan * Copyright (C) 2012 Oleg Moskalenko * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include "bwstring.h" #include "sort.h" bool byte_sort; static wchar_t **wmonths; static char **cmonths; /* initialise months */ void initialise_months(void) { const nl_item item[12] = { ABMON_1, ABMON_2, ABMON_3, ABMON_4, ABMON_5, ABMON_6, ABMON_7, ABMON_8, ABMON_9, ABMON_10, ABMON_11, ABMON_12 }; char *tmp; size_t len; if (mb_cur_max == 1) { if (cmonths == NULL) { char *m; cmonths = sort_malloc(sizeof(char*) * 12); for (int i = 0; i < 12; i++) { cmonths[i] = NULL; tmp = nl_langinfo(item[i]); if (debug_sort) printf("month[%d]=%s\n", i, tmp); if (*tmp == '\0') continue; m = sort_strdup(tmp); len = strlen(tmp); for (unsigned int j = 0; j < len; j++) m[j] = toupper(m[j]); cmonths[i] = m; } } } else { if (wmonths == NULL) { wchar_t *m; wmonths = sort_malloc(sizeof(wchar_t *) * 12); for (int i = 0; i < 12; i++) { wmonths[i] = NULL; tmp = nl_langinfo(item[i]); if (debug_sort) printf("month[%d]=%s\n", i, tmp); if (*tmp == '\0') continue; len = strlen(tmp); m = sort_malloc(SIZEOF_WCHAR_STRING(len + 1)); if (mbstowcs(m, tmp, len) == ((size_t) - 1)) { sort_free(m); continue; } m[len] = L'\0'; for (unsigned int j = 0; j < len; j++) m[j] = towupper(m[j]); wmonths[i] = m; } } } } /* * Compare two wide-character strings */ static int wide_str_coll(const wchar_t *s1, const wchar_t *s2) { int ret; errno = 0; ret = wcscoll(s1, s2); if (errno == EILSEQ) { errno = 0; ret = wcscmp(s1, s2); if (errno != 0) { for (size_t i = 0; ; ++i) { wchar_t c1 = s1[i]; wchar_t c2 = s2[i]; if (c1 == L'\0') return ((c2 == L'\0') ? 0 : -1); if (c2 == L'\0') return (+1); if (c1 == c2) continue; return ((int)(c1 - c2)); } } } return (ret); } /* counterparts of wcs functions */ void bwsprintf(FILE *f, struct bwstring *bws, const char *prefix, const char *suffix) { if (mb_cur_max == 1) fprintf(f, "%s%s%s", prefix, bws->cdata.str, suffix); else fprintf(f, "%s%S%s", prefix, bws->wdata.str, suffix); } const void* bwsrawdata(const struct bwstring *bws) { return (bws->wdata.str); } size_t bwsrawlen(const struct bwstring *bws) { return ((mb_cur_max == 1) ? bws->cdata.len : SIZEOF_WCHAR_STRING(bws->wdata.len)); } size_t bws_memsize(const struct bwstring *bws) { return ((mb_cur_max == 1) ? (bws->cdata.len + 2 + sizeof(struct bwstring)) : (SIZEOF_WCHAR_STRING(bws->wdata.len + 1) + sizeof(struct bwstring))); } void bws_setlen(struct bwstring *bws, size_t newlen) { if (mb_cur_max == 1 && bws && newlen != bws->cdata.len && newlen <= bws->cdata.len) { bws->cdata.len = newlen; bws->cdata.str[newlen] = '\0'; } else if (bws && newlen != bws->wdata.len && newlen <= bws->wdata.len) { bws->wdata.len = newlen; bws->wdata.str[newlen] = L'\0'; } } /* * Allocate a new binary string of specified size */ struct bwstring * bwsalloc(size_t sz) { struct bwstring *ret; if (mb_cur_max == 1) { ret = sort_malloc(sizeof(struct bwstring) + 1 + sz); ret->cdata.len = sz; ret->cdata.str[sz] = '\0'; } else { ret = sort_malloc( sizeof(struct bwstring) + SIZEOF_WCHAR_STRING(sz + 1)); ret->wdata.len = sz; ret->wdata.str[sz] = L'\0'; } return (ret); } /* * Create a copy of binary string. * New string size equals the length of the old string. */ struct bwstring * bwsdup(const struct bwstring *s) { if (s == NULL) return (NULL); else { struct bwstring *ret = bwsalloc(BWSLEN(s)); if (mb_cur_max == 1) memcpy(ret->cdata.str, s->cdata.str, (s->cdata.len)); else memcpy(ret->wdata.str, s->wdata.str, SIZEOF_WCHAR_STRING(s->wdata.len)); return (ret); } } /* * Create a new binary string from a wide character buffer. */ struct bwstring * bwssbdup(const wchar_t *str, size_t len) { if (str == NULL) return ((len == 0) ? bwsalloc(0) : NULL); else { struct bwstring *ret; ret = bwsalloc(len); if (mb_cur_max == 1) for (size_t i = 0; i < len; ++i) ret->cdata.str[i] = (char)str[i]; else memcpy(ret->wdata.str, str, SIZEOF_WCHAR_STRING(len)); return (ret); } } /* * Create a new binary string from a raw binary buffer. */ struct bwstring * bwscsbdup(const unsigned char *str, size_t len) { struct bwstring *ret; ret = bwsalloc(len); if (str) { if (mb_cur_max == 1) memcpy(ret->cdata.str, str, len); else { mbstate_t mbs; const char *s; size_t charlen, chars, cptr; chars = 0; cptr = 0; s = (const char *) str; memset(&mbs, 0, sizeof(mbs)); while (cptr < len) { size_t n = mb_cur_max; if (n > len - cptr) n = len - cptr; charlen = mbrlen(s + cptr, n, &mbs); switch (charlen) { case 0: /* FALLTHROUGH */ case (size_t) -1: /* FALLTHROUGH */ case (size_t) -2: ret->wdata.str[chars++] = (unsigned char) s[cptr]; ++cptr; break; default: n = mbrtowc(ret->wdata.str + (chars++), s + cptr, charlen, &mbs); if ((n == (size_t)-1) || (n == (size_t)-2)) /* NOTREACHED */ err(2, "mbrtowc error"); cptr += charlen; } } ret->wdata.len = chars; ret->wdata.str[ret->wdata.len] = L'\0'; } } return (ret); } /* * De-allocate object memory */ void bwsfree(const struct bwstring *s) { if (s) sort_free(s); } /* * Copy content of src binary string to dst. * If the capacity of the dst string is not sufficient, * then the data is truncated. */ size_t bwscpy(struct bwstring *dst, const struct bwstring *src) { size_t nums = BWSLEN(src); if (nums > BWSLEN(dst)) nums = BWSLEN(dst); if (mb_cur_max == 1) { memcpy(dst->cdata.str, src->cdata.str, nums); dst->cdata.len = nums; dst->cdata.str[dst->cdata.len] = '\0'; } else { memcpy(dst->wdata.str, src->wdata.str, SIZEOF_WCHAR_STRING(nums)); dst->wdata.len = nums; dst->wdata.str[nums] = L'\0'; } return (nums); } /* * Copy content of src binary string to dst, * with specified number of symbols to be copied. * If the capacity of the dst string is not sufficient, * then the data is truncated. */ struct bwstring * bwsncpy(struct bwstring *dst, const struct bwstring *src, size_t size) { size_t nums = BWSLEN(src); if (nums > BWSLEN(dst)) nums = BWSLEN(dst); if (nums > size) nums = size; if (mb_cur_max == 1) { memcpy(dst->cdata.str, src->cdata.str, nums); dst->cdata.len = nums; dst->cdata.str[nums] = '\0'; } else { memcpy(dst->wdata.str, src->wdata.str, SIZEOF_WCHAR_STRING(nums)); dst->wdata.len = nums; dst->wdata.str[nums] = L'\0'; } return (dst); } /* * Copy content of src binary string to dst, * with specified number of symbols to be copied. * An offset value can be specified, from the start of src string. * If the capacity of the dst string is not sufficient, * then the data is truncated. */ struct bwstring * bwsnocpy(struct bwstring *dst, const struct bwstring *src, size_t offset, size_t size) { if (offset >= BWSLEN(src)) { bws_setlen(dst, 0); } else { size_t nums = BWSLEN(src) - offset; if (nums > BWSLEN(dst)) nums = BWSLEN(dst); if (nums > size) nums = size; if (mb_cur_max == 1) { memcpy(dst->cdata.str, src->cdata.str + offset, nums); dst->cdata.len = nums; dst->cdata.str[nums] = '\0'; } else { memcpy(dst->wdata.str, src->wdata.str + offset, SIZEOF_WCHAR_STRING(nums)); dst->wdata.len = nums; dst->wdata.str[nums] = L'\0'; } } return (dst); } /* * Write binary string to the file. * The output is ended either with '\n' (nl == true) * or '\0' (nl == false). */ size_t bwsfwrite(struct bwstring *bws, FILE *f, bool zero_ended) { if (mb_cur_max == 1) { size_t len = bws->cdata.len; if (!zero_ended) { bws->cdata.str[len] = '\n'; if (fwrite(bws->cdata.str, len + 1, 1, f) < 1) err(2, NULL); bws->cdata.str[len] = '\0'; } else if (fwrite(bws->cdata.str, len + 1, 1, f) < 1) err(2, NULL); return (len + 1); } else { wchar_t eols; size_t printed = 0; eols = zero_ended ? btowc('\0') : btowc('\n'); while (printed < BWSLEN(bws)) { const wchar_t *s = bws->wdata.str + printed; if (*s == L'\0') { int nums; nums = fwprintf(f, L"%lc", *s); if (nums != 1) err(2, NULL); ++printed; } else { int nums; nums = fwprintf(f, L"%ls", s); if (nums < 1) err(2, NULL); printed += nums; } } fwprintf(f, L"%lc", eols); return (printed + 1); } } -/* - * Allocate and read a binary string from file. - * The strings are nl-ended or zero-ended, depending on the sort setting. - */ -struct bwstring * -bwsfgetln(FILE *f, size_t *len, bool zero_ended, struct reader_buffer *rb) -{ - wint_t eols; - - eols = zero_ended ? btowc('\0') : btowc('\n'); - - if (!zero_ended && (mb_cur_max > 1)) { - wchar_t *ret; - - ret = fgetwln(f, len); - - if (ret == NULL) { - if (!feof(f)) - err(2, NULL); - return (NULL); - } - if (*len > 0) { - if (ret[*len - 1] == (wchar_t)eols) - --(*len); - } - return (bwssbdup(ret, *len)); - - } else if (!zero_ended && (mb_cur_max == 1)) { - char *ret; - - ret = fgetln(f, len); - - if (ret == NULL) { - if (!feof(f)) - err(2, NULL); - return (NULL); - } - if (*len > 0) { - if (ret[*len - 1] == '\n') - --(*len); - } - return (bwscsbdup((unsigned char *)ret, *len)); - - } else { - *len = 0; - - if (feof(f)) - return (NULL); - - if (2 >= rb->fgetwln_z_buffer_size) { - rb->fgetwln_z_buffer_size += 256; - rb->fgetwln_z_buffer = sort_realloc(rb->fgetwln_z_buffer, - sizeof(wchar_t) * rb->fgetwln_z_buffer_size); - } - rb->fgetwln_z_buffer[*len] = 0; - - if (mb_cur_max == 1) - while (!feof(f)) { - int c; - - c = fgetc(f); - - if (c == EOF) { - if (*len == 0) - return (NULL); - goto line_read_done; - } - if (c == eols) - goto line_read_done; - - if (*len + 1 >= rb->fgetwln_z_buffer_size) { - rb->fgetwln_z_buffer_size += 256; - rb->fgetwln_z_buffer = sort_realloc(rb->fgetwln_z_buffer, - SIZEOF_WCHAR_STRING(rb->fgetwln_z_buffer_size)); - } - - rb->fgetwln_z_buffer[*len] = c; - rb->fgetwln_z_buffer[++(*len)] = 0; - } - else - while (!feof(f)) { - wint_t c; - - c = fgetwc(f); - - if (c == WEOF) { - if (*len == 0) - return (NULL); - goto line_read_done; - } - if (c == eols) - goto line_read_done; - - if (*len + 1 >= rb->fgetwln_z_buffer_size) { - rb->fgetwln_z_buffer_size += 256; - rb->fgetwln_z_buffer = sort_realloc(rb->fgetwln_z_buffer, - SIZEOF_WCHAR_STRING(rb->fgetwln_z_buffer_size)); - } - - rb->fgetwln_z_buffer[*len] = c; - rb->fgetwln_z_buffer[++(*len)] = 0; - } - -line_read_done: - /* we do not count the last 0 */ - return (bwssbdup(rb->fgetwln_z_buffer, *len)); - } -} - int bwsncmp(const struct bwstring *bws1, const struct bwstring *bws2, size_t offset, size_t len) { size_t cmp_len, len1, len2; int res; len1 = BWSLEN(bws1); len2 = BWSLEN(bws2); if (len1 <= offset) { return ((len2 <= offset) ? 0 : -1); } else { if (len2 <= offset) return (+1); else { len1 -= offset; len2 -= offset; cmp_len = len1; if (len2 < cmp_len) cmp_len = len2; if (len < cmp_len) cmp_len = len; if (mb_cur_max == 1) { const char *s1, *s2; s1 = bws1->cdata.str + offset; s2 = bws2->cdata.str + offset; res = memcmp(s1, s2, cmp_len); } else { const wchar_t *s1, *s2; s1 = bws1->wdata.str + offset; s2 = bws2->wdata.str + offset; res = memcmp(s1, s2, SIZEOF_WCHAR_STRING(cmp_len)); } } } if (res == 0) { if (len1 < cmp_len && len1 < len2) res = -1; else if (len2 < cmp_len && len2 < len1) res = +1; } return (res); } int bwscmp(const struct bwstring *bws1, const struct bwstring *bws2, size_t offset) { size_t len1, len2, cmp_len; int res; len1 = BWSLEN(bws1); len2 = BWSLEN(bws2); len1 -= offset; len2 -= offset; cmp_len = len1; if (len2 < cmp_len) cmp_len = len2; res = bwsncmp(bws1, bws2, offset, cmp_len); if (res == 0) { if( len1 < len2) res = -1; else if (len2 < len1) res = +1; } return (res); } int bws_iterator_cmp(bwstring_iterator iter1, bwstring_iterator iter2, size_t len) { wchar_t c1, c2; size_t i; for (i = 0; i < len; ++i) { c1 = bws_get_iter_value(iter1); c2 = bws_get_iter_value(iter2); if (c1 != c2) return (c1 - c2); iter1 = bws_iterator_inc(iter1, 1); iter2 = bws_iterator_inc(iter2, 1); } return (0); } int bwscoll(const struct bwstring *bws1, const struct bwstring *bws2, size_t offset) { size_t len1, len2; len1 = BWSLEN(bws1); len2 = BWSLEN(bws2); if (len1 <= offset) return ((len2 <= offset) ? 0 : -1); else { if (len2 <= offset) return (+1); else { len1 -= offset; len2 -= offset; if (mb_cur_max == 1) { const char *s1, *s2; s1 = bws1->cdata.str + offset; s2 = bws2->cdata.str + offset; if (byte_sort) { int res; if (len1 > len2) { res = memcmp(s1, s2, len2); if (!res) res = +1; } else if (len1 < len2) { res = memcmp(s1, s2, len1); if (!res) res = -1; } else res = memcmp(s1, s2, len1); return (res); } else { int res; size_t i, maxlen; i = 0; maxlen = len1; if (maxlen > len2) maxlen = len2; while (i < maxlen) { /* goto next non-zero part: */ while ((i < maxlen) && !s1[i] && !s2[i]) ++i; if (i >= maxlen) break; if (s1[i] == 0) { if (s2[i] == 0) /* NOTREACHED */ err(2, "bwscoll error 01"); else return (-1); } else if (s2[i] == 0) return (+1); res = strcoll((const char*)(s1 + i), (const char*)(s2 + i)); if (res) return (res); while ((i < maxlen) && s1[i] && s2[i]) ++i; if (i >= maxlen) break; if (s1[i] == 0) { if (s2[i] == 0) { ++i; continue; } else return (-1); } else if (s2[i] == 0) return (+1); else /* NOTREACHED */ err(2, "bwscoll error 02"); } if (len1 < len2) return (-1); else if (len1 > len2) return (+1); return (0); } } else { const wchar_t *s1, *s2; size_t i, maxlen; int res; s1 = bws1->wdata.str + offset; s2 = bws2->wdata.str + offset; i = 0; maxlen = len1; if (maxlen > len2) maxlen = len2; while (i < maxlen) { /* goto next non-zero part: */ while ((i < maxlen) && !s1[i] && !s2[i]) ++i; if (i >= maxlen) break; if (s1[i] == 0) { if (s2[i] == 0) /* NOTREACHED */ err(2, "bwscoll error 1"); else return (-1); } else if (s2[i] == 0) return (+1); res = wide_str_coll(s1 + i, s2 + i); if (res) return (res); while ((i < maxlen) && s1[i] && s2[i]) ++i; if (i >= maxlen) break; if (s1[i] == 0) { if (s2[i] == 0) { ++i; continue; } else return (-1); } else if (s2[i] == 0) return (+1); else /* NOTREACHED */ err(2, "bwscoll error 2"); } if (len1 < len2) return (-1); else if (len1 > len2) return (+1); return (0); } } } } /* * Correction of the system API */ double bwstod(struct bwstring *s0, bool *empty) { double ret; if (mb_cur_max == 1) { char *end, *s; char *ep; s = s0->cdata.str; end = s + s0->cdata.len; ep = NULL; while (isblank(*s) && s < end) ++s; if (!isprint(*s)) { *empty = true; return (0); } ret = strtod((char*)s, &ep); if (ep == s) { *empty = true; return (0); } } else { wchar_t *end, *ep, *s; s = s0->wdata.str; end = s + s0->wdata.len; ep = NULL; while (iswblank(*s) && s < end) ++s; if (!iswprint(*s)) { *empty = true; return (0); } ret = wcstod(s, &ep); if (ep == s) { *empty = true; return (0); } } *empty = false; return (ret); } /* * A helper function for monthcoll. If a line matches * a month name, it returns (number of the month - 1), * while if there is no match, it just return -1. */ int bws_month_score(const struct bwstring *s0) { if (mb_cur_max == 1) { const char *end, *s; s = s0->cdata.str; end = s + s0->cdata.len; while (isblank(*s) && s < end) ++s; for (int i = 11; i >= 0; --i) { if (cmonths[i] && (s == strstr(s, cmonths[i]))) return (i); } } else { const wchar_t *end, *s; s = s0->wdata.str; end = s + s0->wdata.len; while (iswblank(*s) && s < end) ++s; for (int i = 11; i >= 0; --i) { if (wmonths[i] && (s == wcsstr(s, wmonths[i]))) return (i); } } return (-1); } /* * Rips out leading blanks (-b). */ struct bwstring * ignore_leading_blanks(struct bwstring *str) { if (mb_cur_max == 1) { char *dst, *end, *src; src = str->cdata.str; dst = src; end = src + str->cdata.len; while (src < end && isblank(*src)) ++src; if (src != dst) { size_t newlen; newlen = BWSLEN(str) - (src - dst); while (src < end) { *dst = *src; ++dst; ++src; } bws_setlen(str, newlen); } } else { wchar_t *dst, *end, *src; src = str->wdata.str; dst = src; end = src + str->wdata.len; while (src < end && iswblank(*src)) ++src; if (src != dst) { size_t newlen = BWSLEN(str) - (src - dst); while (src < end) { *dst = *src; ++dst; ++src; } bws_setlen(str, newlen); } } return (str); } /* * Rips out nonprinting characters (-i). */ struct bwstring * ignore_nonprinting(struct bwstring *str) { size_t newlen = BWSLEN(str); if (mb_cur_max == 1) { char *dst, *end, *src; char c; src = str->cdata.str; dst = src; end = src + str->cdata.len; while (src < end) { c = *src; if (isprint(c)) { *dst = c; ++dst; ++src; } else { ++src; --newlen; } } } else { wchar_t *dst, *end, *src; wchar_t c; src = str->wdata.str; dst = src; end = src + str->wdata.len; while (src < end) { c = *src; if (iswprint(c)) { *dst = c; ++dst; ++src; } else { ++src; --newlen; } } } bws_setlen(str, newlen); return (str); } /* * Rips out any characters that are not alphanumeric characters * nor blanks (-d). */ struct bwstring * dictionary_order(struct bwstring *str) { size_t newlen = BWSLEN(str); if (mb_cur_max == 1) { char *dst, *end, *src; char c; src = str->cdata.str; dst = src; end = src + str->cdata.len; while (src < end) { c = *src; if (isalnum(c) || isblank(c)) { *dst = c; ++dst; ++src; } else { ++src; --newlen; } } } else { wchar_t *dst, *end, *src; wchar_t c; src = str->wdata.str; dst = src; end = src + str->wdata.len; while (src < end) { c = *src; if (iswalnum(c) || iswblank(c)) { *dst = c; ++dst; ++src; } else { ++src; --newlen; } } } bws_setlen(str, newlen); return (str); } /* * Converts string to lower case(-f). */ struct bwstring * ignore_case(struct bwstring *str) { if (mb_cur_max == 1) { char *end, *s; s = str->cdata.str; end = s + str->cdata.len; while (s < end) { *s = toupper(*s); ++s; } } else { wchar_t *end, *s; s = str->wdata.str; end = s + str->wdata.len; while (s < end) { *s = towupper(*s); ++s; } } return (str); } void bws_disorder_warnx(struct bwstring *s, const char *fn, size_t pos) { if (mb_cur_max == 1) warnx("%s:%zu: disorder: %s", fn, pos + 1, s->cdata.str); else warnx("%s:%zu: disorder: %ls", fn, pos + 1, s->wdata.str); } diff --git a/usr.bin/sort/file.c b/usr.bin/sort/file.c index 1fb02642b5c9..ffcd71e9f991 100644 --- a/usr.bin/sort/file.c +++ b/usr.bin/sort/file.c @@ -1,1597 +1,1603 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (C) 2009 Gabor Kovesdan * Copyright (C) 2012 Oleg Moskalenko * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #if defined(SORT_THREADS) #include #endif #include #include #include #include #include #include #include #include "coll.h" #include "file.h" #include "radixsort.h" unsigned long long free_memory = 1000000; unsigned long long available_free_memory = 1000000; bool use_mmap; const char *tmpdir = "/var/tmp"; const char *compress_program; size_t max_open_files = 16; /* * How much space we read from file at once */ #define READ_CHUNK (4096) /* * File reader structure */ struct file_reader { struct reader_buffer rb; FILE *file; char *fname; - unsigned char *buffer; + char *buffer; unsigned char *mmapaddr; unsigned char *mmapptr; size_t bsz; size_t cbsz; size_t mmapsize; size_t strbeg; int fd; char elsymb; }; /* * Structure to be used in file merge process. */ struct file_header { struct file_reader *fr; struct sort_list_item *si; /* current top line */ size_t file_pos; }; /* * List elements of "cleanable" files list. */ struct CLEANABLE_FILE { char *fn; LIST_ENTRY(CLEANABLE_FILE) files; }; /* * List header of "cleanable" files list. */ static LIST_HEAD(CLEANABLE_FILES,CLEANABLE_FILE) tmp_files; /* * Semaphore to protect the tmp file list. * We use semaphore here because it is signal-safe, according to POSIX. * And semaphore does not require pthread library. */ static sem_t tmp_files_sem; static void mt_sort(struct sort_list *list, int (*sort_func)(void *, size_t, size_t, int (*)(const void *, const void *)), const char* fn); /* * Init tmp files list */ void init_tmp_files(void) { LIST_INIT(&tmp_files); sem_init(&tmp_files_sem, 0, 1); } /* * Save name of a tmp file for signal cleanup */ void tmp_file_atexit(const char *tmp_file) { if (tmp_file) { sem_wait(&tmp_files_sem); struct CLEANABLE_FILE *item = sort_malloc(sizeof(struct CLEANABLE_FILE)); item->fn = sort_strdup(tmp_file); LIST_INSERT_HEAD(&tmp_files, item, files); sem_post(&tmp_files_sem); } } /* * Clear tmp files */ void clear_tmp_files(void) { struct CLEANABLE_FILE *item; sem_wait(&tmp_files_sem); LIST_FOREACH(item,&tmp_files,files) { if ((item) && (item->fn)) unlink(item->fn); } sem_post(&tmp_files_sem); } /* * Check whether a file is a temporary file */ static bool file_is_tmp(const char* fn) { struct CLEANABLE_FILE *item; bool ret = false; if (fn) { sem_wait(&tmp_files_sem); LIST_FOREACH(item,&tmp_files,files) { if ((item) && (item->fn)) if (strcmp(item->fn, fn) == 0) { ret = true; break; } } sem_post(&tmp_files_sem); } return (ret); } /* * Generate new temporary file name */ char * new_tmp_file_name(void) { static size_t tfcounter = 0; static const char *fn = ".bsdsort."; char *ret; size_t sz; sz = strlen(tmpdir) + 1 + strlen(fn) + 32 + 1; ret = sort_malloc(sz); sprintf(ret, "%s/%s%d.%lu", tmpdir, fn, (int) getpid(), (unsigned long)(tfcounter++)); tmp_file_atexit(ret); return (ret); } /* * Initialize file list */ void file_list_init(struct file_list *fl, bool tmp) { if (fl) { fl->count = 0; fl->sz = 0; fl->fns = NULL; fl->tmp = tmp; } } /* * Add a file name to the list */ void file_list_add(struct file_list *fl, const char *fn, bool allocate) { if (fl && fn) { if (fl->count >= fl->sz || (fl->fns == NULL)) { fl->sz = (fl->sz) * 2 + 1; fl->fns = sort_realloc(fl->fns, fl->sz * sizeof(char *)); } fl->fns[fl->count] = allocate ? sort_strdup(fn) : fn; fl->count += 1; } } /* * Populate file list from array of file names */ void file_list_populate(struct file_list *fl, int argc, char **argv, bool allocate) { if (fl && argv) { int i; for (i = 0; i < argc; i++) file_list_add(fl, argv[i], allocate); } } /* * Clean file list data and delete the files, * if this is a list of temporary files */ void file_list_clean(struct file_list *fl) { if (fl) { if (fl->fns) { size_t i; for (i = 0; i < fl->count; i++) { if (fl->fns[i]) { if (fl->tmp) unlink(fl->fns[i]); sort_free(fl->fns[i]); fl->fns[i] = 0; } } sort_free(fl->fns); fl->fns = NULL; } fl->sz = 0; fl->count = 0; fl->tmp = false; } } /* * Init sort list */ void sort_list_init(struct sort_list *l) { if (l) { l->count = 0; l->size = 0; l->memsize = sizeof(struct sort_list); l->list = NULL; } } /* * Add string to sort list */ void sort_list_add(struct sort_list *l, struct bwstring *str) { if (l && str) { size_t indx = l->count; if ((l->list == NULL) || (indx >= l->size)) { size_t newsize = (l->size + 1) + 1024; l->list = sort_realloc(l->list, sizeof(struct sort_list_item*) * newsize); l->memsize += (newsize - l->size) * sizeof(struct sort_list_item*); l->size = newsize; } l->list[indx] = sort_list_item_alloc(); sort_list_item_set(l->list[indx], str); l->memsize += sort_list_item_size(l->list[indx]); l->count += 1; } } /* * Clean sort list data */ void sort_list_clean(struct sort_list *l) { if (l) { if (l->list) { size_t i; for (i = 0; i < l->count; i++) { struct sort_list_item *item; item = l->list[i]; if (item) { sort_list_item_clean(item); sort_free(item); l->list[i] = NULL; } } sort_free(l->list); l->list = NULL; } l->count = 0; l->size = 0; l->memsize = sizeof(struct sort_list); } } /* * Write sort list to file */ void sort_list_dump(struct sort_list *l, const char *fn) { if (l && fn) { FILE *f; f = openfile(fn, "w"); if (f == NULL) err(2, NULL); if (l->list) { size_t i; if (!(sort_opts_vals.uflag)) { for (i = 0; i < l->count; ++i) bwsfwrite(l->list[i]->str, f, sort_opts_vals.zflag); } else { struct sort_list_item *last_printed_item = NULL; struct sort_list_item *item; for (i = 0; i < l->count; ++i) { item = l->list[i]; if ((last_printed_item == NULL) || list_coll(&last_printed_item, &item)) { bwsfwrite(item->str, f, sort_opts_vals.zflag); last_printed_item = item; } } } } closefile(f, fn); } } /* * Checks if the given file is sorted. Stops at the first disorder, * prints the disordered line and returns 1. */ int check(const char *fn) { struct bwstring *s1, *s2, *s1disorder, *s2disorder; struct file_reader *fr; struct keys_array *ka1, *ka2; int res; size_t pos, posdisorder; s1 = s2 = s1disorder = s2disorder = NULL; ka1 = ka2 = NULL; fr = file_reader_init(fn); res = 0; pos = 1; posdisorder = 1; if (fr == NULL) { err(2, NULL); goto end; } s1 = file_reader_readline(fr); if (s1 == NULL) goto end; ka1 = keys_array_alloc(); preproc(s1, ka1); s2 = file_reader_readline(fr); if (s2 == NULL) goto end; ka2 = keys_array_alloc(); preproc(s2, ka2); for (;;) { if (debug_sort) { bwsprintf(stdout, s2, "s1=<", ">"); bwsprintf(stdout, s1, "s2=<", ">"); } int cmp = key_coll(ka2, ka1, 0); if (debug_sort) printf("; cmp1=%d", cmp); if (!cmp && sort_opts_vals.complex_sort && !(sort_opts_vals.uflag) && !(sort_opts_vals.sflag)) { cmp = top_level_str_coll(s2, s1); if (debug_sort) printf("; cmp2=%d", cmp); } if (debug_sort) printf("\n"); if ((sort_opts_vals.uflag && (cmp <= 0)) || (cmp < 0)) { if (!(sort_opts_vals.csilentflag)) { s2disorder = bwsdup(s2); posdisorder = pos; if (debug_sort) s1disorder = bwsdup(s1); } res = 1; goto end; } pos++; clean_keys_array(s1, ka1); sort_free(ka1); ka1 = ka2; ka2 = NULL; bwsfree(s1); s1 = s2; s2 = file_reader_readline(fr); if (s2 == NULL) goto end; ka2 = keys_array_alloc(); preproc(s2, ka2); } end: if (ka1) { clean_keys_array(s1, ka1); sort_free(ka1); } if (s1) bwsfree(s1); if (ka2) { clean_keys_array(s2, ka2); sort_free(ka2); } if (s2) bwsfree(s2); if ((fn == NULL) || (*fn == 0) || (strcmp(fn, "-") == 0)) { for (;;) { s2 = file_reader_readline(fr); if (s2 == NULL) break; bwsfree(s2); } } file_reader_free(fr); if (s2disorder) { bws_disorder_warnx(s2disorder, fn, posdisorder); if (s1disorder) { bws_disorder_warnx(s1disorder, fn, posdisorder); if (s1disorder != s2disorder) bwsfree(s1disorder); } bwsfree(s2disorder); s1disorder = NULL; s2disorder = NULL; } if (res) exit(res); return (0); } /* * Opens a file. If the given filename is "-", stdout will be * opened. */ FILE * openfile(const char *fn, const char *mode) { FILE *file; if (strcmp(fn, "-") == 0) { return ((mode && mode[0] == 'r') ? stdin : stdout); } else { mode_t orig_file_mask = 0; int is_tmp = file_is_tmp(fn); if (is_tmp && (mode[0] == 'w')) orig_file_mask = umask(S_IWGRP | S_IWOTH | S_IRGRP | S_IROTH); if (is_tmp && (compress_program != NULL)) { char *cmd; size_t cmdsz; cmdsz = strlen(fn) + 128; cmd = sort_malloc(cmdsz); fflush(stdout); if (mode[0] == 'r') snprintf(cmd, cmdsz - 1, "cat %s | %s -d", fn, compress_program); else if (mode[0] == 'w') snprintf(cmd, cmdsz - 1, "%s > %s", compress_program, fn); else err(2, "%s", getstr(7)); if ((file = popen(cmd, mode)) == NULL) err(2, NULL); sort_free(cmd); } else if ((file = fopen(fn, mode)) == NULL) err(2, NULL); if (is_tmp && (mode[0] == 'w')) umask(orig_file_mask); } return (file); } /* * Close file */ void closefile(FILE *f, const char *fn) { if (f == NULL) { ; } else if (f == stdin) { ; } else if (f == stdout) { fflush(f); } else { if (file_is_tmp(fn) && compress_program != NULL) { if(pclose(f)<0) err(2,NULL); } else fclose(f); } } /* * Reads a file into the internal buffer. */ struct file_reader * file_reader_init(const char *fsrc) { struct file_reader *ret; if (fsrc == NULL) fsrc = "-"; ret = sort_calloc(1, sizeof(struct file_reader)); ret->elsymb = '\n'; if (sort_opts_vals.zflag) ret->elsymb = 0; ret->fname = sort_strdup(fsrc); if (strcmp(fsrc, "-") && (compress_program == NULL) && use_mmap) { do { struct stat stat_buf; void *addr; size_t sz = 0; int fd, flags; flags = MAP_NOCORE | MAP_NOSYNC; fd = open(fsrc, O_RDONLY); if (fd < 0) err(2, NULL); if (fstat(fd, &stat_buf) < 0) { close(fd); break; } sz = stat_buf.st_size; #if defined(MAP_PREFAULT_READ) flags |= MAP_PREFAULT_READ; #endif addr = mmap(NULL, sz, PROT_READ, flags, fd, 0); if (addr == MAP_FAILED) { close(fd); break; } ret->fd = fd; ret->mmapaddr = addr; ret->mmapsize = sz; ret->mmapptr = ret->mmapaddr; } while (0); } if (ret->mmapaddr == NULL) { ret->file = openfile(fsrc, "r"); if (ret->file == NULL) err(2, NULL); if (strcmp(fsrc, "-")) { ret->cbsz = READ_CHUNK; ret->buffer = sort_malloc(ret->cbsz); ret->bsz = 0; ret->strbeg = 0; ret->bsz = fread(ret->buffer, 1, ret->cbsz, ret->file); if (ret->bsz == 0) { if (ferror(ret->file)) err(2, NULL); } } } return (ret); } struct bwstring * file_reader_readline(struct file_reader *fr) { struct bwstring *ret = NULL; if (fr->mmapaddr) { unsigned char *mmapend; mmapend = fr->mmapaddr + fr->mmapsize; if (fr->mmapptr >= mmapend) return (NULL); else { unsigned char *strend; size_t sz; sz = mmapend - fr->mmapptr; strend = memchr(fr->mmapptr, fr->elsymb, sz); if (strend == NULL) { ret = bwscsbdup(fr->mmapptr, sz); fr->mmapptr = mmapend; } else { ret = bwscsbdup(fr->mmapptr, strend - fr->mmapptr); fr->mmapptr = strend + 1; } } } else if (fr->file != stdin) { - unsigned char *strend; + char *strend; size_t bsz1, remsz, search_start; search_start = 0; remsz = 0; strend = NULL; if (fr->bsz > fr->strbeg) remsz = fr->bsz - fr->strbeg; /* line read cycle */ for (;;) { if (remsz > search_start) strend = memchr(fr->buffer + fr->strbeg + search_start, fr->elsymb, remsz - search_start); else strend = NULL; if (strend) break; if (feof(fr->file)) break; if (fr->bsz != fr->cbsz) /* NOTREACHED */ err(2, "File read software error 1"); if (remsz > (READ_CHUNK >> 1)) { search_start = fr->cbsz - fr->strbeg; fr->cbsz += READ_CHUNK; fr->buffer = sort_realloc(fr->buffer, fr->cbsz); bsz1 = fread(fr->buffer + fr->bsz, 1, READ_CHUNK, fr->file); if (bsz1 == 0) { if (ferror(fr->file)) err(2, NULL); break; } fr->bsz += bsz1; remsz += bsz1; } else { if (remsz > 0 && fr->strbeg>0) bcopy(fr->buffer + fr->strbeg, fr->buffer, remsz); fr->strbeg = 0; search_start = remsz; bsz1 = fread(fr->buffer + remsz, 1, fr->cbsz - remsz, fr->file); if (bsz1 == 0) { if (ferror(fr->file)) err(2, NULL); break; } fr->bsz = remsz + bsz1; remsz = fr->bsz; } } if (strend == NULL) strend = fr->buffer + fr->bsz; if ((fr->buffer + fr->strbeg <= strend) && (fr->strbeg < fr->bsz) && (remsz>0)) ret = bwscsbdup(fr->buffer + fr->strbeg, strend - fr->buffer - fr->strbeg); fr->strbeg = (strend - fr->buffer) + 1; } else { - size_t len = 0; - - ret = bwsfgetln(fr->file, &len, sort_opts_vals.zflag, - &(fr->rb)); + int delim = sort_opts_vals.zflag ? '\0' : '\n'; + ssize_t len = getdelim(&fr->buffer, &fr->bsz, delim, fr->file); + if (len < 0) { + if (!feof(fr->file)) + err(2, NULL); + return (NULL); + } + if (len > 0 && fr->buffer[len - 1] == delim) + len--; + ret = bwscsbdup(fr->buffer, len); } return (ret); } static void file_reader_clean(struct file_reader *fr) { if (fr) { if (fr->mmapaddr) munmap(fr->mmapaddr, fr->mmapsize); if (fr->fd) close(fr->fd); if (fr->buffer) sort_free(fr->buffer); if (fr->file) if (fr->file != stdin) closefile(fr->file, fr->fname); if(fr->fname) sort_free(fr->fname); memset(fr, 0, sizeof(struct file_reader)); } } void file_reader_free(struct file_reader *fr) { if (fr) { file_reader_clean(fr); sort_free(fr); } } int procfile(const char *fsrc, struct sort_list *list, struct file_list *fl) { struct file_reader *fr; fr = file_reader_init(fsrc); if (fr == NULL) err(2, NULL); /* file browse cycle */ for (;;) { struct bwstring *bws; bws = file_reader_readline(fr); if (bws == NULL) break; sort_list_add(list, bws); if (list->memsize >= available_free_memory) { char *fn; fn = new_tmp_file_name(); sort_list_to_file(list, fn); file_list_add(fl, fn, false); sort_list_clean(list); } } file_reader_free(fr); return (0); } /* * Compare file headers. Files with EOF always go to the end of the list. */ static int file_header_cmp(struct file_header *f1, struct file_header *f2) { if (f1 == f2) return (0); else { if (f1->fr == NULL) { return ((f2->fr == NULL) ? 0 : +1); } else if (f2->fr == NULL) return (-1); else { int ret; ret = list_coll(&(f1->si), &(f2->si)); if (!ret) return ((f1->file_pos < f2->file_pos) ? -1 : +1); return (ret); } } } /* * Allocate and init file header structure */ static void file_header_init(struct file_header **fh, const char *fn, size_t file_pos) { if (fh && fn) { struct bwstring *line; *fh = sort_malloc(sizeof(struct file_header)); (*fh)->file_pos = file_pos; (*fh)->fr = file_reader_init(fn); if ((*fh)->fr == NULL) { perror(fn); err(2, "%s", getstr(8)); } line = file_reader_readline((*fh)->fr); if (line == NULL) { file_reader_free((*fh)->fr); (*fh)->fr = NULL; (*fh)->si = NULL; } else { (*fh)->si = sort_list_item_alloc(); sort_list_item_set((*fh)->si, line); } } } /* * Close file */ static void file_header_close(struct file_header **fh) { if (fh && *fh) { if ((*fh)->fr) { file_reader_free((*fh)->fr); (*fh)->fr = NULL; } if ((*fh)->si) { sort_list_item_clean((*fh)->si); sort_free((*fh)->si); (*fh)->si = NULL; } sort_free(*fh); *fh = NULL; } } /* * Swap two array elements */ static void file_header_swap(struct file_header **fh, size_t i1, size_t i2) { struct file_header *tmp; tmp = fh[i1]; fh[i1] = fh[i2]; fh[i2] = tmp; } /* heap algorithm ==>> */ /* * See heap sort algorithm * "Raises" last element to its right place */ static void file_header_heap_swim(struct file_header **fh, size_t indx) { if (indx > 0) { size_t parent_index; parent_index = (indx - 1) >> 1; if (file_header_cmp(fh[indx], fh[parent_index]) < 0) { /* swap child and parent and continue */ file_header_swap(fh, indx, parent_index); file_header_heap_swim(fh, parent_index); } } } /* * Sink the top element to its correct position */ static void file_header_heap_sink(struct file_header **fh, size_t indx, size_t size) { size_t left_child_index; size_t right_child_index; left_child_index = indx + indx + 1; right_child_index = left_child_index + 1; if (left_child_index < size) { size_t min_child_index; min_child_index = left_child_index; if ((right_child_index < size) && (file_header_cmp(fh[left_child_index], fh[right_child_index]) > 0)) min_child_index = right_child_index; if (file_header_cmp(fh[indx], fh[min_child_index]) > 0) { file_header_swap(fh, indx, min_child_index); file_header_heap_sink(fh, min_child_index, size); } } } /* <<== heap algorithm */ /* * Adds element to the "left" end */ static void file_header_list_rearrange_from_header(struct file_header **fh, size_t size) { file_header_heap_sink(fh, 0, size); } /* * Adds element to the "right" end */ static void file_header_list_push(struct file_header *f, struct file_header **fh, size_t size) { fh[size++] = f; file_header_heap_swim(fh, size - 1); } struct last_printed { struct bwstring *str; }; /* * Prints the current line of the file */ static void file_header_print(struct file_header *fh, FILE *f_out, struct last_printed *lp) { if (fh && fh->fr && f_out && fh->si && fh->si->str) { if (sort_opts_vals.uflag) { if ((lp->str == NULL) || (str_list_coll(lp->str, &(fh->si)))) { bwsfwrite(fh->si->str, f_out, sort_opts_vals.zflag); if (lp->str) bwsfree(lp->str); lp->str = bwsdup(fh->si->str); } } else bwsfwrite(fh->si->str, f_out, sort_opts_vals.zflag); } } /* * Read next line */ static void file_header_read_next(struct file_header *fh) { if (fh && fh->fr) { struct bwstring *tmp; tmp = file_reader_readline(fh->fr); if (tmp == NULL) { file_reader_free(fh->fr); fh->fr = NULL; if (fh->si) { sort_list_item_clean(fh->si); sort_free(fh->si); fh->si = NULL; } } else { if (fh->si == NULL) fh->si = sort_list_item_alloc(); sort_list_item_set(fh->si, tmp); } } } /* * Merge array of "files headers" */ static void file_headers_merge(size_t fnum, struct file_header **fh, FILE *f_out) { struct last_printed lp; size_t i; memset(&lp, 0, sizeof(lp)); /* * construct the initial sort structure */ for (i = 0; i < fnum; i++) file_header_list_push(fh[i], fh, i); while (fh[0]->fr) { /* unfinished files are always in front */ /* output the smallest line: */ file_header_print(fh[0], f_out, &lp); /* read a new line, if possible: */ file_header_read_next(fh[0]); /* re-arrange the list: */ file_header_list_rearrange_from_header(fh, fnum); } if (lp.str) bwsfree(lp.str); } /* * Merges the given files into the output file, which can be * stdout. */ static void merge_files_array(size_t argc, const char **argv, const char *fn_out) { if (argv && fn_out) { struct file_header **fh; FILE *f_out; size_t i; f_out = openfile(fn_out, "w"); if (f_out == NULL) err(2, NULL); fh = sort_malloc((argc + 1) * sizeof(struct file_header *)); for (i = 0; i < argc; i++) file_header_init(fh + i, argv[i], (size_t) i); file_headers_merge(argc, fh, f_out); for (i = 0; i < argc; i++) file_header_close(fh + i); sort_free(fh); closefile(f_out, fn_out); } } /* * Shrinks the file list until its size smaller than max number of opened files */ static int shrink_file_list(struct file_list *fl) { if ((fl == NULL) || (size_t) (fl->count) < max_open_files) return (0); else { struct file_list new_fl; size_t indx = 0; file_list_init(&new_fl, true); while (indx < fl->count) { char *fnew; size_t num; num = fl->count - indx; fnew = new_tmp_file_name(); if ((size_t) num >= max_open_files) num = max_open_files - 1; merge_files_array(num, fl->fns + indx, fnew); if (fl->tmp) { size_t i; for (i = 0; i < num; i++) unlink(fl->fns[indx + i]); } file_list_add(&new_fl, fnew, false); indx += num; } fl->tmp = false; /* already taken care of */ file_list_clean(fl); fl->count = new_fl.count; fl->fns = new_fl.fns; fl->sz = new_fl.sz; fl->tmp = new_fl.tmp; return (1); } } /* * Merge list of files */ void merge_files(struct file_list *fl, const char *fn_out) { if (fl && fn_out) { while (shrink_file_list(fl)); merge_files_array(fl->count, fl->fns, fn_out); } } static const char * get_sort_method_name(int sm) { if (sm == SORT_MERGESORT) return "mergesort"; else if (sort_opts_vals.sort_method == SORT_RADIXSORT) return "radixsort"; else if (sort_opts_vals.sort_method == SORT_HEAPSORT) return "heapsort"; else return "quicksort"; } /* * Wrapper for qsort */ static int sort_qsort(void *list, size_t count, size_t elem_size, int (*cmp_func)(const void *, const void *)) { qsort(list, count, elem_size, cmp_func); return (0); } /* * Sort list of lines and writes it to the file */ void sort_list_to_file(struct sort_list *list, const char *outfile) { struct sort_mods *sm = &(keys[0].sm); if (!(sm->Mflag) && !(sm->Rflag) && !(sm->Vflag) && !(sm->gflag) && !(sm->hflag) && !(sm->nflag)) { if ((sort_opts_vals.sort_method == SORT_DEFAULT) && byte_sort) sort_opts_vals.sort_method = SORT_RADIXSORT; } else if (sort_opts_vals.sort_method == SORT_RADIXSORT) err(2, "%s", getstr(9)); /* * to handle stable sort and the unique cases in the * right order, we need stable basic algorithm */ if (sort_opts_vals.sflag) { switch (sort_opts_vals.sort_method){ case SORT_MERGESORT: break; case SORT_RADIXSORT: break; case SORT_DEFAULT: sort_opts_vals.sort_method = SORT_MERGESORT; break; default: errx(2, "%s", getstr(10)); } } if (sort_opts_vals.sort_method == SORT_DEFAULT) sort_opts_vals.sort_method = DEFAULT_SORT_ALGORITHM; if (debug_sort) printf("sort_method=%s\n", get_sort_method_name(sort_opts_vals.sort_method)); switch (sort_opts_vals.sort_method){ case SORT_RADIXSORT: rxsort(list->list, list->count); sort_list_dump(list, outfile); break; case SORT_MERGESORT: mt_sort(list, mergesort, outfile); break; case SORT_HEAPSORT: mt_sort(list, heapsort, outfile); break; case SORT_QSORT: mt_sort(list, sort_qsort, outfile); break; default: mt_sort(list, DEFAULT_SORT_FUNC, outfile); break; } } /******************* MT SORT ************************/ #if defined(SORT_THREADS) /* semaphore to count threads */ static sem_t mtsem; /* current system sort function */ static int (*g_sort_func)(void *, size_t, size_t, int(*)(const void *, const void *)); /* * Sort cycle thread (in multi-threaded mode) */ static void* mt_sort_thread(void* arg) { struct sort_list *list = arg; g_sort_func(list->list, list->count, sizeof(struct sort_list_item *), (int(*)(const void *, const void *)) list_coll); sem_post(&mtsem); return (arg); } /* * Compare sub-lists. Empty sub-lists always go to the end of the list. */ static int sub_list_cmp(struct sort_list *l1, struct sort_list *l2) { if (l1 == l2) return (0); else { if (l1->count == 0) { return ((l2->count == 0) ? 0 : +1); } else if (l2->count == 0) { return (-1); } else { int ret; ret = list_coll(&(l1->list[0]), &(l2->list[0])); if (!ret) return ((l1->sub_list_pos < l2->sub_list_pos) ? -1 : +1); return (ret); } } } /* * Swap two array elements */ static void sub_list_swap(struct sort_list **sl, size_t i1, size_t i2) { struct sort_list *tmp; tmp = sl[i1]; sl[i1] = sl[i2]; sl[i2] = tmp; } /* heap algorithm ==>> */ /* * See heap sort algorithm * "Raises" last element to its right place */ static void sub_list_swim(struct sort_list **sl, size_t indx) { if (indx > 0) { size_t parent_index; parent_index = (indx - 1) >> 1; if (sub_list_cmp(sl[indx], sl[parent_index]) < 0) { /* swap child and parent and continue */ sub_list_swap(sl, indx, parent_index); sub_list_swim(sl, parent_index); } } } /* * Sink the top element to its correct position */ static void sub_list_sink(struct sort_list **sl, size_t indx, size_t size) { size_t left_child_index; size_t right_child_index; left_child_index = indx + indx + 1; right_child_index = left_child_index + 1; if (left_child_index < size) { size_t min_child_index; min_child_index = left_child_index; if ((right_child_index < size) && (sub_list_cmp(sl[left_child_index], sl[right_child_index]) > 0)) min_child_index = right_child_index; if (sub_list_cmp(sl[indx], sl[min_child_index]) > 0) { sub_list_swap(sl, indx, min_child_index); sub_list_sink(sl, min_child_index, size); } } } /* <<== heap algorithm */ /* * Adds element to the "right" end */ static void sub_list_push(struct sort_list *s, struct sort_list **sl, size_t size) { sl[size++] = s; sub_list_swim(sl, size - 1); } struct last_printed_item { struct sort_list_item *item; }; /* * Prints the current line of the file */ static void sub_list_header_print(struct sort_list *sl, FILE *f_out, struct last_printed_item *lp) { if (sl && sl->count && f_out && sl->list[0]->str) { if (sort_opts_vals.uflag) { if ((lp->item == NULL) || (list_coll(&(lp->item), &(sl->list[0])))) { bwsfwrite(sl->list[0]->str, f_out, sort_opts_vals.zflag); lp->item = sl->list[0]; } } else bwsfwrite(sl->list[0]->str, f_out, sort_opts_vals.zflag); } } /* * Read next line */ static void sub_list_next(struct sort_list *sl) { if (sl && sl->count) { sl->list += 1; sl->count -= 1; } } /* * Merge sub-lists to a file */ static void merge_sub_lists(struct sort_list **sl, size_t n, FILE* f_out) { struct last_printed_item lp; size_t i; memset(&lp,0,sizeof(lp)); /* construct the initial list: */ for (i = 0; i < n; i++) sub_list_push(sl[i], sl, i); while (sl[0]->count) { /* unfinished lists are always in front */ /* output the smallest line: */ sub_list_header_print(sl[0], f_out, &lp); /* move to a new line, if possible: */ sub_list_next(sl[0]); /* re-arrange the list: */ sub_list_sink(sl, 0, n); } } /* * Merge sub-lists to a file */ static void merge_list_parts(struct sort_list **parts, size_t n, const char *fn) { FILE* f_out; f_out = openfile(fn,"w"); merge_sub_lists(parts, n, f_out); closefile(f_out, fn); } #endif /* defined(SORT_THREADS) */ /* * Multi-threaded sort algorithm "driver" */ static void mt_sort(struct sort_list *list, int(*sort_func)(void *, size_t, size_t, int(*)(const void *, const void *)), const char* fn) { #if defined(SORT_THREADS) if (nthreads < 2 || list->count < MT_SORT_THRESHOLD) { size_t nthreads_save = nthreads; nthreads = 1; #endif /* if single thread or small data, do simple sort */ sort_func(list->list, list->count, sizeof(struct sort_list_item *), (int(*)(const void *, const void *)) list_coll); sort_list_dump(list, fn); #if defined(SORT_THREADS) nthreads = nthreads_save; } else { /* multi-threaded sort */ struct sort_list **parts; size_t avgsize, cstart, i; /* array of sub-lists */ parts = sort_malloc(sizeof(struct sort_list*) * nthreads); cstart = 0; avgsize = list->count / nthreads; /* set global system sort function */ g_sort_func = sort_func; /* set sublists */ for (i = 0; i < nthreads; ++i) { size_t sz = 0; parts[i] = sort_malloc(sizeof(struct sort_list)); parts[i]->list = list->list + cstart; parts[i]->memsize = 0; parts[i]->sub_list_pos = i; sz = (i == nthreads - 1) ? list->count - cstart : avgsize; parts[i]->count = sz; parts[i]->size = parts[i]->count; cstart += sz; } /* init threads counting semaphore */ sem_init(&mtsem, 0, 0); /* start threads */ for (i = 0; i < nthreads; ++i) { pthread_t pth; pthread_attr_t attr; pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_DETACHED); for (;;) { int res = pthread_create(&pth, &attr, mt_sort_thread, parts[i]); if (res >= 0) break; if (errno == EAGAIN) { pthread_yield(); continue; } err(2, NULL); } pthread_attr_destroy(&attr); } /* wait for threads completion */ for (i = 0; i < nthreads; ++i) { sem_wait(&mtsem); } /* destroy the semaphore - we do not need it anymore */ sem_destroy(&mtsem); /* merge sorted sub-lists to the file */ merge_list_parts(parts, nthreads, fn); /* free sub-lists data */ for (i = 0; i < nthreads; ++i) { sort_free(parts[i]); } sort_free(parts); } #endif /* defined(SORT_THREADS) */ }