Index: head/usr.bin/sort/coll.c =================================================================== --- head/usr.bin/sort/coll.c (revision 346174) +++ head/usr.bin/sort/coll.c (revision 346175) @@ -1,1300 +1,1323 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (C) 2009 Gabor Kovesdan * Copyright (C) 2012 Oleg Moskalenko * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include "coll.h" #include "vsort.h" struct key_specs *keys; size_t keys_num = 0; wint_t symbol_decimal_point = L'.'; /* there is no default thousands separator in collate rules: */ wint_t symbol_thousands_sep = 0; wint_t symbol_negative_sign = L'-'; wint_t symbol_positive_sign = L'+'; static int wstrcoll(struct key_value *kv1, struct key_value *kv2, size_t offset); static int gnumcoll(struct key_value*, struct key_value *, size_t offset); static int monthcoll(struct key_value*, struct key_value *, size_t offset); static int numcoll(struct key_value*, struct key_value *, size_t offset); static int hnumcoll(struct key_value*, struct key_value *, size_t offset); static int randomcoll(struct key_value*, struct key_value *, size_t offset); static int versioncoll(struct key_value*, struct key_value *, size_t offset); /* * Allocate keys array */ struct keys_array * keys_array_alloc(void) { struct keys_array *ka; size_t sz; sz = keys_array_size(); ka = sort_malloc(sz); memset(ka, 0, sz); return (ka); } /* * Calculate whether we need key hint space */ static size_t key_hint_size(void) { return (need_hint ? sizeof(struct key_hint) : 0); } /* * Calculate keys array size */ size_t keys_array_size(void) { return (keys_num * (sizeof(struct key_value) + key_hint_size())); } /* * Clean data of keys array */ void clean_keys_array(const struct bwstring *s, struct keys_array *ka) { if (ka) { for (size_t i = 0; i < keys_num; ++i) { const struct key_value *kv; kv = get_key_from_keys_array(ka, i); if (kv->k && kv->k != s) bwsfree(kv->k); } memset(ka, 0, keys_array_size()); } } /* * Get pointer to a key value in the keys set */ struct key_value * get_key_from_keys_array(struct keys_array *ka, size_t ind) { return ((struct key_value *)((caddr_t)ka->key + ind * (sizeof(struct key_value) + key_hint_size()))); } /* * Set value of a key in the keys set */ void set_key_on_keys_array(struct keys_array *ka, struct bwstring *s, size_t ind) { if (ka && keys_num > ind) { struct key_value *kv; kv = get_key_from_keys_array(ka, ind); if (kv->k && kv->k != s) bwsfree(kv->k); kv->k = s; } } /* * Initialize a sort list item */ struct sort_list_item * sort_list_item_alloc(void) { struct sort_list_item *si; size_t sz; sz = sizeof(struct sort_list_item) + keys_array_size(); si = sort_malloc(sz); memset(si, 0, sz); return (si); } size_t sort_list_item_size(struct sort_list_item *si) { size_t ret = 0; if (si) { ret = sizeof(struct sort_list_item) + keys_array_size(); if (si->str) ret += bws_memsize(si->str); for (size_t i = 0; i < keys_num; ++i) { const struct key_value *kv; kv = get_key_from_keys_array(&si->ka, i); if (kv->k != si->str) ret += bws_memsize(kv->k); } } return (ret); } /* * Calculate key for a sort list item */ static void sort_list_item_make_key(struct sort_list_item *si) { preproc(si->str, &(si->ka)); } /* * Set value of a sort list item. * Return combined string and keys memory size. */ void sort_list_item_set(struct sort_list_item *si, struct bwstring *str) { if (si) { clean_keys_array(si->str, &(si->ka)); if (si->str) { if (si->str == str) { /* we are trying to reset the same string */ return; } else { bwsfree(si->str); si->str = NULL; } } si->str = str; sort_list_item_make_key(si); } } /* * De-allocate a sort list item object memory */ void sort_list_item_clean(struct sort_list_item *si) { if (si) { clean_keys_array(si->str, &(si->ka)); if (si->str) { bwsfree(si->str); si->str = NULL; } } } /* * Skip columns according to specs */ static size_t skip_cols_to_start(const struct bwstring *s, size_t cols, size_t start, bool skip_blanks, bool *empty_key) { if (cols < 1) return (BWSLEN(s) + 1); if (skip_blanks) while (start < BWSLEN(s) && iswblank(BWS_GET(s,start))) ++start; while (start < BWSLEN(s) && cols > 1) { --cols; ++start; } if (start >= BWSLEN(s)) *empty_key = true; return (start); } /* * Skip fields according to specs */ static size_t skip_fields_to_start(const struct bwstring *s, size_t fields, bool *empty_field) { if (fields < 2) { if (BWSLEN(s) == 0) *empty_field = true; return (0); } else if (!(sort_opts_vals.tflag)) { size_t cpos = 0; bool pb = true; while (cpos < BWSLEN(s)) { bool isblank; isblank = iswblank(BWS_GET(s, cpos)); if (isblank && !pb) { --fields; if (fields <= 1) return (cpos); } pb = isblank; ++cpos; } if (fields > 1) *empty_field = true; return (cpos); } else { size_t cpos = 0; while (cpos < BWSLEN(s)) { if (BWS_GET(s,cpos) == (wchar_t)sort_opts_vals.field_sep) { --fields; if (fields <= 1) return (cpos + 1); } ++cpos; } if (fields > 1) *empty_field = true; return (cpos); } } /* * Find fields start */ static void find_field_start(const struct bwstring *s, struct key_specs *ks, size_t *field_start, size_t *key_start, bool *empty_field, bool *empty_key) { *field_start = skip_fields_to_start(s, ks->f1, empty_field); if (!*empty_field) *key_start = skip_cols_to_start(s, ks->c1, *field_start, ks->pos1b, empty_key); else *empty_key = true; } /* * Find end key position */ static size_t find_field_end(const struct bwstring *s, struct key_specs *ks) { size_t f2, next_field_start, pos_end; bool empty_field, empty_key; empty_field = false; empty_key = false; f2 = ks->f2; if (f2 == 0) return (BWSLEN(s) + 1); else { if (ks->c2 == 0) { next_field_start = skip_fields_to_start(s, f2 + 1, &empty_field); if ((next_field_start > 0) && sort_opts_vals.tflag && ((wchar_t)sort_opts_vals.field_sep == BWS_GET(s, next_field_start - 1))) --next_field_start; } else next_field_start = skip_fields_to_start(s, f2, &empty_field); } if (empty_field || (next_field_start >= BWSLEN(s))) return (BWSLEN(s) + 1); if (ks->c2) { pos_end = skip_cols_to_start(s, ks->c2, next_field_start, ks->pos2b, &empty_key); if (pos_end < BWSLEN(s)) ++pos_end; } else pos_end = next_field_start; return (pos_end); } /* * Cut a field according to the key specs */ static struct bwstring * cut_field(const struct bwstring *s, struct key_specs *ks) { struct bwstring *ret = NULL; if (s && ks) { size_t field_start, key_end, key_start, sz; bool empty_field, empty_key; field_start = 0; key_start = 0; empty_field = false; empty_key = false; find_field_start(s, ks, &field_start, &key_start, &empty_field, &empty_key); if (empty_key) sz = 0; else { key_end = find_field_end(s, ks); sz = (key_end < key_start) ? 0 : (key_end - key_start); } ret = bwsalloc(sz); if (sz) bwsnocpy(ret, s, key_start, sz); } else ret = bwsalloc(0); return (ret); } /* * Preprocesses a line applying the necessary transformations * specified by command line options and returns the preprocessed * string, which can be used to compare. */ int preproc(struct bwstring *s, struct keys_array *ka) { if (sort_opts_vals.kflag) for (size_t i = 0; i < keys_num; i++) { struct bwstring *key; struct key_specs *kspecs; struct sort_mods *sm; kspecs = &(keys[i]); key = cut_field(s, kspecs); sm = &(kspecs->sm); if (sm->dflag) key = dictionary_order(key); else if (sm->iflag) key = ignore_nonprinting(key); if (sm->fflag || sm->Mflag) key = ignore_case(key); set_key_on_keys_array(ka, key, i); } else { struct bwstring *ret = NULL; struct sort_mods *sm = default_sort_mods; if (sm->bflag) { if (ret == NULL) ret = bwsdup(s); ret = ignore_leading_blanks(ret); } if (sm->dflag) { if (ret == NULL) ret = bwsdup(s); ret = dictionary_order(ret); } else if (sm->iflag) { if (ret == NULL) ret = bwsdup(s); ret = ignore_nonprinting(ret); } if (sm->fflag || sm->Mflag) { if (ret == NULL) ret = bwsdup(s); ret = ignore_case(ret); } if (ret == NULL) set_key_on_keys_array(ka, s, 0); else set_key_on_keys_array(ka, ret, 0); } return 0; } cmpcoll_t get_sort_func(struct sort_mods *sm) { if (sm->nflag) return (numcoll); else if (sm->hflag) return (hnumcoll); else if (sm->gflag) return (gnumcoll); else if (sm->Mflag) return (monthcoll); else if (sm->Rflag) return (randomcoll); else if (sm->Vflag) return (versioncoll); else return (wstrcoll); } /* * Compares the given strings. Returns a positive number if * the first precedes the second, a negative number if the second is * the preceding one, and zero if they are equal. This function calls * the underlying collate functions, which done the actual comparison. */ int key_coll(struct keys_array *ps1, struct keys_array *ps2, size_t offset) { struct key_value *kv1, *kv2; struct sort_mods *sm; int res = 0; for (size_t i = 0; i < keys_num; ++i) { kv1 = get_key_from_keys_array(ps1, i); kv2 = get_key_from_keys_array(ps2, i); sm = &(keys[i].sm); if (sm->rflag) res = sm->func(kv2, kv1, offset); else res = sm->func(kv1, kv2, offset); if (res) break; /* offset applies to only the first key */ offset = 0; } return (res); } /* * Compare two strings. * Plain symbol-by-symbol comparison. */ int top_level_str_coll(const struct bwstring *s1, const struct bwstring *s2) { if (default_sort_mods->rflag) { const struct bwstring *tmp; tmp = s1; s1 = s2; s2 = tmp; } return (bwscoll(s1, s2, 0)); } /* * Compare a string and a sort list item, according to the sort specs. */ int str_list_coll(struct bwstring *str1, struct sort_list_item **ss2) { struct keys_array *ka1; int ret = 0; ka1 = keys_array_alloc(); preproc(str1, ka1); sort_list_item_make_key(*ss2); if (debug_sort) { bwsprintf(stdout, str1, "; s1=<", ">"); bwsprintf(stdout, (*ss2)->str, ", s2=<", ">"); } ret = key_coll(ka1, &((*ss2)->ka), 0); if (debug_sort) printf("; cmp1=%d", ret); clean_keys_array(str1, ka1); sort_free(ka1); if ((ret == 0) && !(sort_opts_vals.sflag) && sort_opts_vals.complex_sort) { ret = top_level_str_coll(str1, ((*ss2)->str)); if (debug_sort) printf("; cmp2=%d", ret); } if (debug_sort) printf("\n"); return (ret); } /* * Compare two sort list items, according to the sort specs. */ int list_coll_offset(struct sort_list_item **ss1, struct sort_list_item **ss2, size_t offset) { int ret; ret = key_coll(&((*ss1)->ka), &((*ss2)->ka), offset); if (debug_sort) { if (offset) printf("; offset=%d", (int) offset); bwsprintf(stdout, ((*ss1)->str), "; s1=<", ">"); bwsprintf(stdout, ((*ss2)->str), ", s2=<", ">"); printf("; cmp1=%d\n", ret); } if (ret) return (ret); if (!(sort_opts_vals.sflag) && sort_opts_vals.complex_sort) { ret = top_level_str_coll(((*ss1)->str), ((*ss2)->str)); if (debug_sort) printf("; cmp2=%d\n", ret); } return (ret); } /* * Compare two sort list items, according to the sort specs. */ int list_coll(struct sort_list_item **ss1, struct sort_list_item **ss2) { return (list_coll_offset(ss1, ss2, 0)); } #define LSCDEF(N) \ static int \ list_coll_##N(struct sort_list_item **ss1, struct sort_list_item **ss2) \ { \ \ return (list_coll_offset(ss1, ss2, N)); \ } LSCDEF(1) LSCDEF(2) LSCDEF(3) LSCDEF(4) LSCDEF(5) LSCDEF(6) LSCDEF(7) LSCDEF(8) LSCDEF(9) LSCDEF(10) LSCDEF(11) LSCDEF(12) LSCDEF(13) LSCDEF(14) LSCDEF(15) LSCDEF(16) LSCDEF(17) LSCDEF(18) LSCDEF(19) LSCDEF(20) listcoll_t get_list_call_func(size_t offset) { static const listcoll_t lsarray[] = { list_coll, list_coll_1, list_coll_2, list_coll_3, list_coll_4, list_coll_5, list_coll_6, list_coll_7, list_coll_8, list_coll_9, list_coll_10, list_coll_11, list_coll_12, list_coll_13, list_coll_14, list_coll_15, list_coll_16, list_coll_17, list_coll_18, list_coll_19, list_coll_20 }; if (offset <= 20) return (lsarray[offset]); return (list_coll); } /* * Compare two sort list items, only by their original string. */ int list_coll_by_str_only(struct sort_list_item **ss1, struct sort_list_item **ss2) { return (top_level_str_coll(((*ss1)->str), ((*ss2)->str))); } /* * Maximum size of a number in the string (before or after decimal point) */ #define MAX_NUM_SIZE (128) /* * Set suffix value */ static void setsuffix(wchar_t c, unsigned char *si) { switch (c){ case L'k': case L'K': *si = 1; break; case L'M': *si = 2; break; case L'G': *si = 3; break; case L'T': *si = 4; break; case L'P': *si = 5; break; case L'E': *si = 6; break; case L'Z': *si = 7; break; case L'Y': *si = 8; break; default: *si = 0; } } /* * Read string s and parse the string into a fixed-decimal-point number. * sign equals -1 if the number is negative (explicit plus is not allowed, * according to GNU sort's "info sort". * The number part before decimal point is in the smain, after the decimal * point is in sfrac, tail is the pointer to the remainder of the string. */ static int read_number(struct bwstring *s0, int *sign, wchar_t *smain, size_t *main_len, wchar_t *sfrac, size_t *frac_len, unsigned char *si) { bwstring_iterator s; s = bws_begin(s0); /* always end the fraction with zero, even if we have no fraction */ sfrac[0] = 0; while (iswblank(bws_get_iter_value(s))) s = bws_iterator_inc(s, 1); if (bws_get_iter_value(s) == (wchar_t)symbol_negative_sign) { *sign = -1; s = bws_iterator_inc(s, 1); } // This is '0', not '\0', do not change this while (iswdigit(bws_get_iter_value(s)) && (bws_get_iter_value(s) == L'0')) s = bws_iterator_inc(s, 1); while (bws_get_iter_value(s) && *main_len < MAX_NUM_SIZE) { if (iswdigit(bws_get_iter_value(s))) { smain[*main_len] = bws_get_iter_value(s); s = bws_iterator_inc(s, 1); *main_len += 1; } else if (symbol_thousands_sep && (bws_get_iter_value(s) == (wchar_t)symbol_thousands_sep)) s = bws_iterator_inc(s, 1); else break; } smain[*main_len] = 0; if (bws_get_iter_value(s) == (wchar_t)symbol_decimal_point) { s = bws_iterator_inc(s, 1); while (iswdigit(bws_get_iter_value(s)) && *frac_len < MAX_NUM_SIZE) { sfrac[*frac_len] = bws_get_iter_value(s); s = bws_iterator_inc(s, 1); *frac_len += 1; } sfrac[*frac_len] = 0; while (*frac_len > 0 && sfrac[*frac_len - 1] == L'0') { --(*frac_len); sfrac[*frac_len] = L'\0'; } } setsuffix(bws_get_iter_value(s),si); if ((*main_len + *frac_len) == 0) *sign = 0; return (0); } /* * Implements string sort. */ static int wstrcoll(struct key_value *kv1, struct key_value *kv2, size_t offset) { if (debug_sort) { if (offset) printf("; offset=%d\n", (int) offset); bwsprintf(stdout, kv1->k, "; k1=<", ">"); printf("(%zu)", BWSLEN(kv1->k)); bwsprintf(stdout, kv2->k, ", k2=<", ">"); printf("(%zu)", BWSLEN(kv2->k)); } return (bwscoll(kv1->k, kv2->k, offset)); } /* * Compare two suffixes */ static inline int cmpsuffix(unsigned char si1, unsigned char si2) { return ((char)si1 - (char)si2); } /* * Implements numeric sort for -n and -h. */ static int numcoll_impl(struct key_value *kv1, struct key_value *kv2, size_t offset __unused, bool use_suffix) { struct bwstring *s1, *s2; wchar_t sfrac1[MAX_NUM_SIZE + 1], sfrac2[MAX_NUM_SIZE + 1]; wchar_t smain1[MAX_NUM_SIZE + 1], smain2[MAX_NUM_SIZE + 1]; int cmp_res, sign1, sign2; size_t frac1, frac2, main1, main2; unsigned char SI1, SI2; bool e1, e2, key1_read, key2_read; s1 = kv1->k; s2 = kv2->k; sign1 = sign2 = 0; main1 = main2 = 0; frac1 = frac2 = 0; key1_read = key2_read = false; if (debug_sort) { bwsprintf(stdout, s1, "; k1=<", ">"); bwsprintf(stdout, s2, ", k2=<", ">"); } if (s1 == s2) return (0); if (kv1->hint->status == HS_UNINITIALIZED) { /* read the number from the string */ read_number(s1, &sign1, smain1, &main1, sfrac1, &frac1, &SI1); key1_read = true; kv1->hint->v.nh.n1 = wcstoull(smain1, NULL, 10); if(main1 < 1 && frac1 < 1) kv1->hint->v.nh.empty=true; kv1->hint->v.nh.si = SI1; kv1->hint->status = (kv1->hint->v.nh.n1 != ULLONG_MAX) ? HS_INITIALIZED : HS_ERROR; kv1->hint->v.nh.neg = (sign1 < 0) ? true : false; } if (kv2->hint->status == HS_UNINITIALIZED) { /* read the number from the string */ read_number(s2, &sign2, smain2, &main2, sfrac2, &frac2,&SI2); key2_read = true; kv2->hint->v.nh.n1 = wcstoull(smain2, NULL, 10); if(main2 < 1 && frac2 < 1) kv2->hint->v.nh.empty=true; kv2->hint->v.nh.si = SI2; kv2->hint->status = (kv2->hint->v.nh.n1 != ULLONG_MAX) ? HS_INITIALIZED : HS_ERROR; kv2->hint->v.nh.neg = (sign2 < 0) ? true : false; } if (kv1->hint->status == HS_INITIALIZED && kv2->hint->status == HS_INITIALIZED) { unsigned long long n1, n2; bool neg1, neg2; e1 = kv1->hint->v.nh.empty; e2 = kv2->hint->v.nh.empty; if (e1 && e2) return (0); neg1 = kv1->hint->v.nh.neg; neg2 = kv2->hint->v.nh.neg; if (neg1 && !neg2) return (-1); if (neg2 && !neg1) return (+1); if (e1) return (neg2 ? +1 : -1); else if (e2) return (neg1 ? -1 : +1); if (use_suffix) { cmp_res = cmpsuffix(kv1->hint->v.nh.si, kv2->hint->v.nh.si); if (cmp_res) return (neg1 ? -cmp_res : cmp_res); } n1 = kv1->hint->v.nh.n1; n2 = kv2->hint->v.nh.n1; if (n1 < n2) return (neg1 ? +1 : -1); else if (n1 > n2) return (neg1 ? -1 : +1); } /* read the numbers from the strings */ if (!key1_read) read_number(s1, &sign1, smain1, &main1, sfrac1, &frac1, &SI1); if (!key2_read) read_number(s2, &sign2, smain2, &main2, sfrac2, &frac2, &SI2); e1 = ((main1 + frac1) == 0); e2 = ((main2 + frac2) == 0); if (e1 && e2) return (0); /* we know the result if the signs are different */ if (sign1 < 0 && sign2 >= 0) return (-1); if (sign1 >= 0 && sign2 < 0) return (+1); if (e1) return ((sign2 < 0) ? +1 : -1); else if (e2) return ((sign1 < 0) ? -1 : +1); if (use_suffix) { cmp_res = cmpsuffix(SI1, SI2); if (cmp_res) return ((sign1 < 0) ? -cmp_res : cmp_res); } /* if both numbers are empty assume that the strings are equal */ if (main1 < 1 && main2 < 1 && frac1 < 1 && frac2 < 1) return (0); /* * if the main part is of different size, we know the result * (because the leading zeros are removed) */ if (main1 < main2) cmp_res = -1; else if (main1 > main2) cmp_res = +1; /* if the sizes are equal then simple non-collate string compare gives the correct result */ else cmp_res = wcscmp(smain1, smain2); /* check fraction */ if (!cmp_res) cmp_res = wcscmp(sfrac1, sfrac2); if (!cmp_res) return (0); /* reverse result if the signs are negative */ if (sign1 < 0 && sign2 < 0) cmp_res = -cmp_res; return (cmp_res); } /* * Implements numeric sort (-n). */ static int numcoll(struct key_value *kv1, struct key_value *kv2, size_t offset) { return (numcoll_impl(kv1, kv2, offset, false)); } /* * Implements 'human' numeric sort (-h). */ static int hnumcoll(struct key_value *kv1, struct key_value *kv2, size_t offset) { return (numcoll_impl(kv1, kv2, offset, true)); } +/* Use hint space to memoize md5 computations, at least. */ +static void +randomcoll_init_hint(struct key_value *kv, void *hash) +{ + + memcpy(kv->hint->v.Rh.cached, hash, sizeof(kv->hint->v.Rh.cached)); + kv->hint->status = HS_INITIALIZED; +} + /* * Implements random sort (-R). */ static int randomcoll(struct key_value *kv1, struct key_value *kv2, size_t offset __unused) { struct bwstring *s1, *s2; MD5_CTX ctx1, ctx2; unsigned char hash1[MD5_DIGEST_LENGTH], hash2[MD5_DIGEST_LENGTH]; + int cmp; s1 = kv1->k; s2 = kv2->k; if (debug_sort) { bwsprintf(stdout, s1, "; k1=<", ">"); bwsprintf(stdout, s2, ", k2=<", ">"); } if (s1 == s2) return (0); + if (kv1->hint->status == HS_INITIALIZED && + kv2->hint->status == HS_INITIALIZED) { + cmp = memcmp(kv1->hint->v.Rh.cached, + kv2->hint->v.Rh.cached, sizeof(kv1->hint->v.Rh.cached)); + if (cmp != 0) + return (cmp); + } + memcpy(&ctx1, &md5_ctx, sizeof(MD5_CTX)); memcpy(&ctx2, &md5_ctx, sizeof(MD5_CTX)); MD5Update(&ctx1, bwsrawdata(s1), bwsrawlen(s1)); MD5Update(&ctx2, bwsrawdata(s2), bwsrawlen(s2)); MD5Final(hash1, &ctx1); MD5Final(hash2, &ctx2); + + if (kv1->hint->status == HS_UNINITIALIZED) + randomcoll_init_hint(kv1, hash1); + if (kv2->hint->status == HS_UNINITIALIZED) + randomcoll_init_hint(kv2, hash2); return (memcmp(hash1, hash2, sizeof(hash1))); } /* * Implements version sort (-V). */ static int versioncoll(struct key_value *kv1, struct key_value *kv2, size_t offset __unused) { struct bwstring *s1, *s2; s1 = kv1->k; s2 = kv2->k; if (debug_sort) { bwsprintf(stdout, s1, "; k1=<", ">"); bwsprintf(stdout, s2, ", k2=<", ">"); } if (s1 == s2) return (0); return (vcmp(s1, s2)); } /* * Check for minus infinity */ static inline bool huge_minus(double d, int err1) { if (err1 == ERANGE) if (d == -HUGE_VAL || d == -HUGE_VALF || d == -HUGE_VALL) return (+1); return (0); } /* * Check for plus infinity */ static inline bool huge_plus(double d, int err1) { if (err1 == ERANGE) if (d == HUGE_VAL || d == HUGE_VALF || d == HUGE_VALL) return (+1); return (0); } /* * Check whether a function is a NAN */ static bool is_nan(double d) { return ((d == NAN) || (isnan(d))); } /* * Compare two NANs */ static int cmp_nans(double d1, double d2) { if (d1 < d2) return (-1); if (d1 > d2) return (+1); return (0); } /* * Implements general numeric sort (-g). */ static int gnumcoll(struct key_value *kv1, struct key_value *kv2, size_t offset __unused) { double d1, d2; int err1, err2; bool empty1, empty2, key1_read, key2_read; d1 = d2 = 0; err1 = err2 = 0; key1_read = key2_read = false; if (debug_sort) { bwsprintf(stdout, kv1->k, "; k1=<", ">"); bwsprintf(stdout, kv2->k, "; k2=<", ">"); } if (kv1->hint->status == HS_UNINITIALIZED) { errno = 0; d1 = bwstod(kv1->k, &empty1); err1 = errno; if (empty1) kv1->hint->v.gh.notnum = true; else if (err1 == 0) { kv1->hint->v.gh.d = d1; kv1->hint->v.gh.nan = is_nan(d1); kv1->hint->status = HS_INITIALIZED; } else kv1->hint->status = HS_ERROR; key1_read = true; } if (kv2->hint->status == HS_UNINITIALIZED) { errno = 0; d2 = bwstod(kv2->k, &empty2); err2 = errno; if (empty2) kv2->hint->v.gh.notnum = true; else if (err2 == 0) { kv2->hint->v.gh.d = d2; kv2->hint->v.gh.nan = is_nan(d2); kv2->hint->status = HS_INITIALIZED; } else kv2->hint->status = HS_ERROR; key2_read = true; } if (kv1->hint->status == HS_INITIALIZED && kv2->hint->status == HS_INITIALIZED) { if (kv1->hint->v.gh.notnum) return ((kv2->hint->v.gh.notnum) ? 0 : -1); else if (kv2->hint->v.gh.notnum) return (+1); if (kv1->hint->v.gh.nan) return ((kv2->hint->v.gh.nan) ? cmp_nans(kv1->hint->v.gh.d, kv2->hint->v.gh.d) : -1); else if (kv2->hint->v.gh.nan) return (+1); d1 = kv1->hint->v.gh.d; d2 = kv2->hint->v.gh.d; if (d1 < d2) return (-1); else if (d1 > d2) return (+1); else return (0); } if (!key1_read) { errno = 0; d1 = bwstod(kv1->k, &empty1); err1 = errno; } if (!key2_read) { errno = 0; d2 = bwstod(kv2->k, &empty2); err2 = errno; } /* Non-value case: */ if (empty1) return (empty2 ? 0 : -1); else if (empty2) return (+1); /* NAN case */ if (is_nan(d1)) return (is_nan(d2) ? cmp_nans(d1, d2) : -1); else if (is_nan(d2)) return (+1); /* Infinities */ if (err1 == ERANGE || err2 == ERANGE) { /* Minus infinity case */ if (huge_minus(d1, err1)) { if (huge_minus(d2, err2)) { if (d1 < d2) return (-1); if (d1 > d2) return (+1); return (0); } else return (-1); } else if (huge_minus(d2, err2)) { if (huge_minus(d1, err1)) { if (d1 < d2) return (-1); if (d1 > d2) return (+1); return (0); } else return (+1); } /* Plus infinity case */ if (huge_plus(d1, err1)) { if (huge_plus(d2, err2)) { if (d1 < d2) return (-1); if (d1 > d2) return (+1); return (0); } else return (+1); } else if (huge_plus(d2, err2)) { if (huge_plus(d1, err1)) { if (d1 < d2) return (-1); if (d1 > d2) return (+1); return (0); } else return (-1); } } if (d1 < d2) return (-1); if (d1 > d2) return (+1); return (0); } /* * Implements month sort (-M). */ static int monthcoll(struct key_value *kv1, struct key_value *kv2, size_t offset __unused) { int val1, val2; bool key1_read, key2_read; val1 = val2 = 0; key1_read = key2_read = false; if (debug_sort) { bwsprintf(stdout, kv1->k, "; k1=<", ">"); bwsprintf(stdout, kv2->k, "; k2=<", ">"); } if (kv1->hint->status == HS_UNINITIALIZED) { kv1->hint->v.Mh.m = bws_month_score(kv1->k); key1_read = true; kv1->hint->status = HS_INITIALIZED; } if (kv2->hint->status == HS_UNINITIALIZED) { kv2->hint->v.Mh.m = bws_month_score(kv2->k); key2_read = true; kv2->hint->status = HS_INITIALIZED; } if (kv1->hint->status == HS_INITIALIZED) { val1 = kv1->hint->v.Mh.m; key1_read = true; } if (kv2->hint->status == HS_INITIALIZED) { val2 = kv2->hint->v.Mh.m; key2_read = true; } if (!key1_read) val1 = bws_month_score(kv1->k); if (!key2_read) val2 = bws_month_score(kv2->k); if (val1 == val2) { return (0); } if (val1 < val2) return (-1); return (+1); } Index: head/usr.bin/sort/coll.h =================================================================== --- head/usr.bin/sort/coll.h (revision 346174) +++ head/usr.bin/sort/coll.h (revision 346175) @@ -1,170 +1,182 @@ /* $FreeBSD$ */ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (C) 2009 Gabor Kovesdan * Copyright (C) 2012 Oleg Moskalenko * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #if !defined(__COLL_H__) #define __COLL_H__ #include "bwstring.h" #include "sort.h" /* * Sort hint data for -n */ struct n_hint { unsigned long long n1; unsigned char si; bool empty; bool neg; }; /* * Sort hint data for -g */ struct g_hint { double d; bool nan; bool notnum; }; /* * Sort hint data for -M */ struct M_hint { int m; }; /* + * Sort hint data for -R + * + * This stores the first 12 bytes of the digest rather than the full output to + * avoid increasing the size of the 'key_hint' object via the 'v' union. + */ +struct R_hint +{ + unsigned char cached[12]; +}; + +/* * Status of a sort hint object */ typedef enum { HS_ERROR = -1, HS_UNINITIALIZED = 0, HS_INITIALIZED = 1 } hint_status; /* * Sort hint object */ struct key_hint { hint_status status; union { struct n_hint nh; struct g_hint gh; struct M_hint Mh; + struct R_hint Rh; } v; }; /* * Key value */ struct key_value { struct bwstring *k; /* key string */ struct key_hint hint[0]; /* key sort hint */ } __packed; /* * Set of keys container object. */ struct keys_array { struct key_value key[0]; }; /* * Parsed -k option data */ struct key_specs { struct sort_mods sm; size_t c1; size_t c2; size_t f1; size_t f2; bool pos1b; bool pos2b; }; /* * Single entry in sort list. */ struct sort_list_item { struct bwstring *str; struct keys_array ka; }; /* * Function type, used to compare two list objects */ typedef int (*listcoll_t)(struct sort_list_item **ss1, struct sort_list_item **ss2); extern struct key_specs *keys; extern size_t keys_num; /* * Main localised symbols. These must be wint_t as they may hold WEOF. */ extern wint_t symbol_decimal_point; extern wint_t symbol_thousands_sep; extern wint_t symbol_negative_sign; extern wint_t symbol_positive_sign; /* funcs */ cmpcoll_t get_sort_func(struct sort_mods *sm); struct keys_array *keys_array_alloc(void); size_t keys_array_size(void); struct key_value *get_key_from_keys_array(struct keys_array *ka, size_t ind); void set_key_on_keys_array(struct keys_array *ka, struct bwstring *s, size_t ind); void clean_keys_array(const struct bwstring *s, struct keys_array *ka); struct sort_list_item *sort_list_item_alloc(void); void sort_list_item_set(struct sort_list_item *si, struct bwstring *str); void sort_list_item_clean(struct sort_list_item *si); size_t sort_list_item_size(struct sort_list_item *si); int preproc(struct bwstring *s, struct keys_array *ka); int top_level_str_coll(const struct bwstring *, const struct bwstring *); int key_coll(struct keys_array *ks1, struct keys_array *ks2, size_t offset); int str_list_coll(struct bwstring *str1, struct sort_list_item **ss2); int list_coll_by_str_only(struct sort_list_item **ss1, struct sort_list_item **ss2); int list_coll(struct sort_list_item **ss1, struct sort_list_item **ss2); int list_coll_offset(struct sort_list_item **ss1, struct sort_list_item **ss2, size_t offset); listcoll_t get_list_call_func(size_t offset); #endif /* __COLL_H__ */ Index: head/usr.bin/sort/sort.c =================================================================== --- head/usr.bin/sort/sort.c (revision 346174) +++ head/usr.bin/sort/sort.c (revision 346175) @@ -1,1337 +1,1338 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (C) 2009 Gabor Kovesdan * Copyright (C) 2012 Oleg Moskalenko * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "coll.h" #include "file.h" #include "sort.h" #ifndef WITHOUT_NLS #include nl_catd catalog; #endif #define OPTIONS "bcCdfghik:Mmno:RrsS:t:T:uVz" static bool need_random; MD5_CTX md5_ctx; /* * Default messages to use when NLS is disabled or no catalogue * is found. */ const char *nlsstr[] = { "", /* 1*/"mutually exclusive flags", /* 2*/"extra argument not allowed with -c", /* 3*/"Unknown feature", /* 4*/"Wrong memory buffer specification", /* 5*/"0 field in key specs", /* 6*/"0 column in key specs", /* 7*/"Wrong file mode", /* 8*/"Cannot open file for reading", /* 9*/"Radix sort cannot be used with these sort options", /*10*/"The chosen sort method cannot be used with stable and/or unique sort", /*11*/"Invalid key position", /*12*/"Usage: %s [-bcCdfigMmnrsuz] [-kPOS1[,POS2] ... ] " "[+POS1 [-POS2]] [-S memsize] [-T tmpdir] [-t separator] " "[-o outfile] [--batch-size size] [--files0-from file] " "[--heapsort] [--mergesort] [--radixsort] [--qsort] " "[--mmap] " #if defined(SORT_THREADS) "[--parallel thread_no] " #endif "[--human-numeric-sort] " "[--version-sort] [--random-sort [--random-source file]] " "[--compress-program program] [file ...]\n" }; struct sort_opts sort_opts_vals; bool debug_sort; bool need_hint; #if defined(SORT_THREADS) unsigned int ncpu = 1; size_t nthreads = 1; #endif static bool gnusort_numeric_compatibility; static struct sort_mods default_sort_mods_object; struct sort_mods * const default_sort_mods = &default_sort_mods_object; static bool print_symbols_on_debug; /* * Arguments from file (when file0-from option is used: */ static size_t argc_from_file0 = (size_t)-1; static char **argv_from_file0; /* * Placeholder symbols for options which have no single-character equivalent */ enum { SORT_OPT = CHAR_MAX + 1, HELP_OPT, FF_OPT, BS_OPT, VERSION_OPT, DEBUG_OPT, #if defined(SORT_THREADS) PARALLEL_OPT, #endif RANDOMSOURCE_OPT, COMPRESSPROGRAM_OPT, QSORT_OPT, MERGESORT_OPT, HEAPSORT_OPT, RADIXSORT_OPT, MMAP_OPT }; #define NUMBER_OF_MUTUALLY_EXCLUSIVE_FLAGS 6 static const char mutually_exclusive_flags[NUMBER_OF_MUTUALLY_EXCLUSIVE_FLAGS] = { 'M', 'n', 'g', 'R', 'h', 'V' }; static struct option long_options[] = { { "batch-size", required_argument, NULL, BS_OPT }, { "buffer-size", required_argument, NULL, 'S' }, { "check", optional_argument, NULL, 'c' }, { "check=silent|quiet", optional_argument, NULL, 'C' }, { "compress-program", required_argument, NULL, COMPRESSPROGRAM_OPT }, { "debug", no_argument, NULL, DEBUG_OPT }, { "dictionary-order", no_argument, NULL, 'd' }, { "field-separator", required_argument, NULL, 't' }, { "files0-from", required_argument, NULL, FF_OPT }, { "general-numeric-sort", no_argument, NULL, 'g' }, { "heapsort", no_argument, NULL, HEAPSORT_OPT }, { "help",no_argument, NULL, HELP_OPT }, { "human-numeric-sort", no_argument, NULL, 'h' }, { "ignore-leading-blanks", no_argument, NULL, 'b' }, { "ignore-case", no_argument, NULL, 'f' }, { "ignore-nonprinting", no_argument, NULL, 'i' }, { "key", required_argument, NULL, 'k' }, { "merge", no_argument, NULL, 'm' }, { "mergesort", no_argument, NULL, MERGESORT_OPT }, { "mmap", no_argument, NULL, MMAP_OPT }, { "month-sort", no_argument, NULL, 'M' }, { "numeric-sort", no_argument, NULL, 'n' }, { "output", required_argument, NULL, 'o' }, #if defined(SORT_THREADS) { "parallel", required_argument, NULL, PARALLEL_OPT }, #endif { "qsort", no_argument, NULL, QSORT_OPT }, { "radixsort", no_argument, NULL, RADIXSORT_OPT }, { "random-sort", no_argument, NULL, 'R' }, { "random-source", required_argument, NULL, RANDOMSOURCE_OPT }, { "reverse", no_argument, NULL, 'r' }, { "sort", required_argument, NULL, SORT_OPT }, { "stable", no_argument, NULL, 's' }, { "temporary-directory",required_argument, NULL, 'T' }, { "unique", no_argument, NULL, 'u' }, { "version", no_argument, NULL, VERSION_OPT }, { "version-sort",no_argument, NULL, 'V' }, { "zero-terminated", no_argument, NULL, 'z' }, { NULL, no_argument, NULL, 0 } }; void fix_obsolete_keys(int *argc, char **argv); /* * Check where sort modifier is present */ static bool sort_modifier_empty(struct sort_mods *sm) { if (sm == NULL) return (true); return (!(sm->Mflag || sm->Vflag || sm->nflag || sm->gflag || sm->rflag || sm->Rflag || sm->hflag || sm->dflag || sm->fflag)); } /* * Print out usage text. */ static void usage(bool opt_err) { FILE *out; out = opt_err ? stderr : stdout; fprintf(out, getstr(12), getprogname()); if (opt_err) exit(2); exit(0); } /* * Read input file names from a file (file0-from option). */ static void read_fns_from_file0(const char *fn) { FILE *f; char *line = NULL; size_t linesize = 0; ssize_t linelen; if (fn == NULL) return; f = fopen(fn, "r"); if (f == NULL) err(2, "%s", fn); while ((linelen = getdelim(&line, &linesize, '\0', f)) != -1) { if (*line != '\0') { if (argc_from_file0 == (size_t) - 1) argc_from_file0 = 0; ++argc_from_file0; argv_from_file0 = sort_realloc(argv_from_file0, argc_from_file0 * sizeof(char *)); if (argv_from_file0 == NULL) err(2, NULL); argv_from_file0[argc_from_file0 - 1] = line; } else { free(line); } line = NULL; linesize = 0; } if (ferror(f)) err(2, "%s: getdelim", fn); closefile(f, fn); } /* * Check how much RAM is available for the sort. */ static void set_hw_params(void) { long pages, psize; #if defined(SORT_THREADS) ncpu = 1; #endif pages = sysconf(_SC_PHYS_PAGES); if (pages < 1) { perror("sysconf pages"); pages = 1; } psize = sysconf(_SC_PAGESIZE); if (psize < 1) { perror("sysconf psize"); psize = 4096; } #if defined(SORT_THREADS) ncpu = (unsigned int)sysconf(_SC_NPROCESSORS_ONLN); if (ncpu < 1) ncpu = 1; else if(ncpu > 32) ncpu = 32; nthreads = ncpu; #endif free_memory = (unsigned long long) pages * (unsigned long long) psize; available_free_memory = free_memory / 2; if (available_free_memory < 1024) available_free_memory = 1024; } /* * Convert "plain" symbol to wide symbol, with default value. */ static void conv_mbtowc(wchar_t *wc, const char *c, const wchar_t def) { if (wc && c) { int res; res = mbtowc(wc, c, MB_CUR_MAX); if (res < 1) *wc = def; } } /* * Set current locale symbols. */ static void set_locale(void) { struct lconv *lc; const char *locale; setlocale(LC_ALL, ""); lc = localeconv(); if (lc) { /* obtain LC_NUMERIC info */ /* Convert to wide char form */ conv_mbtowc(&symbol_decimal_point, lc->decimal_point, symbol_decimal_point); conv_mbtowc(&symbol_thousands_sep, lc->thousands_sep, symbol_thousands_sep); conv_mbtowc(&symbol_positive_sign, lc->positive_sign, symbol_positive_sign); conv_mbtowc(&symbol_negative_sign, lc->negative_sign, symbol_negative_sign); } if (getenv("GNUSORT_NUMERIC_COMPATIBILITY")) gnusort_numeric_compatibility = true; locale = setlocale(LC_COLLATE, NULL); if (locale) { char *tmpl; const char *cclocale; tmpl = sort_strdup(locale); cclocale = setlocale(LC_COLLATE, "C"); if (cclocale && !strcmp(cclocale, tmpl)) byte_sort = true; else { const char *pclocale; pclocale = setlocale(LC_COLLATE, "POSIX"); if (pclocale && !strcmp(pclocale, tmpl)) byte_sort = true; } setlocale(LC_COLLATE, tmpl); sort_free(tmpl); } } /* * Set directory temporary files. */ static void set_tmpdir(void) { char *td; td = getenv("TMPDIR"); if (td != NULL) tmpdir = sort_strdup(td); } /* * Parse -S option. */ static unsigned long long parse_memory_buffer_value(const char *value) { if (value == NULL) return (available_free_memory); else { char *endptr; unsigned long long membuf; endptr = NULL; errno = 0; membuf = strtoll(value, &endptr, 10); if (errno != 0) { warn("%s",getstr(4)); membuf = available_free_memory; } else { switch (*endptr){ case 'Y': membuf *= 1024; /* FALLTHROUGH */ case 'Z': membuf *= 1024; /* FALLTHROUGH */ case 'E': membuf *= 1024; /* FALLTHROUGH */ case 'P': membuf *= 1024; /* FALLTHROUGH */ case 'T': membuf *= 1024; /* FALLTHROUGH */ case 'G': membuf *= 1024; /* FALLTHROUGH */ case 'M': membuf *= 1024; /* FALLTHROUGH */ case '\0': case 'K': membuf *= 1024; /* FALLTHROUGH */ case 'b': break; case '%': membuf = (available_free_memory * membuf) / 100; break; default: warnc(EINVAL, "%s", optarg); membuf = available_free_memory; } } return (membuf); } } /* * Signal handler that clears the temporary files. */ static void sig_handler(int sig __unused, siginfo_t *siginfo __unused, void *context __unused) { clear_tmp_files(); exit(-1); } /* * Set signal handler on panic signals. */ static void set_signal_handler(void) { struct sigaction sa; memset(&sa, 0, sizeof(sa)); sa.sa_sigaction = &sig_handler; sa.sa_flags = SA_SIGINFO; if (sigaction(SIGTERM, &sa, NULL) < 0) { perror("sigaction"); return; } if (sigaction(SIGHUP, &sa, NULL) < 0) { perror("sigaction"); return; } if (sigaction(SIGINT, &sa, NULL) < 0) { perror("sigaction"); return; } if (sigaction(SIGQUIT, &sa, NULL) < 0) { perror("sigaction"); return; } if (sigaction(SIGABRT, &sa, NULL) < 0) { perror("sigaction"); return; } if (sigaction(SIGBUS, &sa, NULL) < 0) { perror("sigaction"); return; } if (sigaction(SIGSEGV, &sa, NULL) < 0) { perror("sigaction"); return; } if (sigaction(SIGUSR1, &sa, NULL) < 0) { perror("sigaction"); return; } if (sigaction(SIGUSR2, &sa, NULL) < 0) { perror("sigaction"); return; } } /* * Print "unknown" message and exit with status 2. */ static void unknown(const char *what) { errx(2, "%s: %s", getstr(3), what); } /* * Check whether contradictory input options are used. */ static void check_mutually_exclusive_flags(char c, bool *mef_flags) { int fo_index, mec; bool found_others, found_this; found_others = found_this = false; fo_index = 0; for (int i = 0; i < NUMBER_OF_MUTUALLY_EXCLUSIVE_FLAGS; i++) { mec = mutually_exclusive_flags[i]; if (mec != c) { if (mef_flags[i]) { if (found_this) errx(1, "%c:%c: %s", c, mec, getstr(1)); found_others = true; fo_index = i; } } else { if (found_others) errx(1, "%c:%c: %s", c, mutually_exclusive_flags[fo_index], getstr(1)); mef_flags[i] = true; found_this = true; } } } /* * Initialise sort opts data. */ static void set_sort_opts(void) { memset(&default_sort_mods_object, 0, sizeof(default_sort_mods_object)); memset(&sort_opts_vals, 0, sizeof(sort_opts_vals)); default_sort_mods_object.func = get_sort_func(&default_sort_mods_object); } /* * Set a sort modifier on a sort modifiers object. */ static bool set_sort_modifier(struct sort_mods *sm, int c) { if (sm == NULL) return (true); switch (c){ case 'b': sm->bflag = true; break; case 'd': sm->dflag = true; break; case 'f': sm->fflag = true; break; case 'g': sm->gflag = true; need_hint = true; break; case 'i': sm->iflag = true; break; case 'R': sm->Rflag = true; + need_hint = true; need_random = true; break; case 'M': initialise_months(); sm->Mflag = true; need_hint = true; break; case 'n': sm->nflag = true; need_hint = true; print_symbols_on_debug = true; break; case 'r': sm->rflag = true; break; case 'V': sm->Vflag = true; break; case 'h': sm->hflag = true; need_hint = true; print_symbols_on_debug = true; break; default: return (false); } sort_opts_vals.complex_sort = true; sm->func = get_sort_func(sm); return (true); } /* * Parse POS in -k option. */ static int parse_pos(const char *s, struct key_specs *ks, bool *mef_flags, bool second) { regmatch_t pmatch[4]; regex_t re; char *c, *f; const char *sregexp = "^([0-9]+)(\\.[0-9]+)?([bdfirMngRhV]+)?$"; size_t len, nmatch; int ret; ret = -1; nmatch = 4; c = f = NULL; if (regcomp(&re, sregexp, REG_EXTENDED) != 0) return (-1); if (regexec(&re, s, nmatch, pmatch, 0) != 0) goto end; if (pmatch[0].rm_eo <= pmatch[0].rm_so) goto end; if (pmatch[1].rm_eo <= pmatch[1].rm_so) goto end; len = pmatch[1].rm_eo - pmatch[1].rm_so; f = sort_malloc((len + 1) * sizeof(char)); strncpy(f, s + pmatch[1].rm_so, len); f[len] = '\0'; if (second) { errno = 0; ks->f2 = (size_t) strtoul(f, NULL, 10); if (errno != 0) err(2, "-k"); if (ks->f2 == 0) { warn("%s",getstr(5)); goto end; } } else { errno = 0; ks->f1 = (size_t) strtoul(f, NULL, 10); if (errno != 0) err(2, "-k"); if (ks->f1 == 0) { warn("%s",getstr(5)); goto end; } } if (pmatch[2].rm_eo > pmatch[2].rm_so) { len = pmatch[2].rm_eo - pmatch[2].rm_so - 1; c = sort_malloc((len + 1) * sizeof(char)); strncpy(c, s + pmatch[2].rm_so + 1, len); c[len] = '\0'; if (second) { errno = 0; ks->c2 = (size_t) strtoul(c, NULL, 10); if (errno != 0) err(2, "-k"); } else { errno = 0; ks->c1 = (size_t) strtoul(c, NULL, 10); if (errno != 0) err(2, "-k"); if (ks->c1 == 0) { warn("%s",getstr(6)); goto end; } } } else { if (second) ks->c2 = 0; else ks->c1 = 1; } if (pmatch[3].rm_eo > pmatch[3].rm_so) { regoff_t i = 0; for (i = pmatch[3].rm_so; i < pmatch[3].rm_eo; i++) { check_mutually_exclusive_flags(s[i], mef_flags); if (s[i] == 'b') { if (second) ks->pos2b = true; else ks->pos1b = true; } else if (!set_sort_modifier(&(ks->sm), s[i])) goto end; } } ret = 0; end: if (c) sort_free(c); if (f) sort_free(f); regfree(&re); return (ret); } /* * Parse -k option value. */ static int parse_k(const char *s, struct key_specs *ks) { int ret = -1; bool mef_flags[NUMBER_OF_MUTUALLY_EXCLUSIVE_FLAGS] = { false, false, false, false, false, false }; if (s && *s) { char *sptr; sptr = strchr(s, ','); if (sptr) { size_t size1; char *pos1, *pos2; size1 = sptr - s; if (size1 < 1) return (-1); pos1 = sort_malloc((size1 + 1) * sizeof(char)); strncpy(pos1, s, size1); pos1[size1] = '\0'; ret = parse_pos(pos1, ks, mef_flags, false); sort_free(pos1); if (ret < 0) return (ret); pos2 = sort_strdup(sptr + 1); ret = parse_pos(pos2, ks, mef_flags, true); sort_free(pos2); } else ret = parse_pos(s, ks, mef_flags, false); } return (ret); } /* * Parse POS in +POS -POS option. */ static int parse_pos_obs(const char *s, int *nf, int *nc, char* sopts) { regex_t re; regmatch_t pmatch[4]; char *c, *f; const char *sregexp = "^([0-9]+)(\\.[0-9]+)?([A-Za-z]+)?$"; int ret; size_t len, nmatch; ret = -1; nmatch = 4; c = f = NULL; *nc = *nf = 0; if (regcomp(&re, sregexp, REG_EXTENDED) != 0) return (-1); if (regexec(&re, s, nmatch, pmatch, 0) != 0) goto end; if (pmatch[0].rm_eo <= pmatch[0].rm_so) goto end; if (pmatch[1].rm_eo <= pmatch[1].rm_so) goto end; len = pmatch[1].rm_eo - pmatch[1].rm_so; f = sort_malloc((len + 1) * sizeof(char)); strncpy(f, s + pmatch[1].rm_so, len); f[len] = '\0'; errno = 0; *nf = (size_t) strtoul(f, NULL, 10); if (errno != 0) errx(2, "%s", getstr(11)); if (pmatch[2].rm_eo > pmatch[2].rm_so) { len = pmatch[2].rm_eo - pmatch[2].rm_so - 1; c = sort_malloc((len + 1) * sizeof(char)); strncpy(c, s + pmatch[2].rm_so + 1, len); c[len] = '\0'; errno = 0; *nc = (size_t) strtoul(c, NULL, 10); if (errno != 0) errx(2, "%s", getstr(11)); } if (pmatch[3].rm_eo > pmatch[3].rm_so) { len = pmatch[3].rm_eo - pmatch[3].rm_so; strncpy(sopts, s + pmatch[3].rm_so, len); sopts[len] = '\0'; } ret = 0; end: if (c) sort_free(c); if (f) sort_free(f); regfree(&re); return (ret); } /* * "Translate" obsolete +POS1 -POS2 syntax into new -kPOS1,POS2 syntax */ void fix_obsolete_keys(int *argc, char **argv) { char sopt[129]; for (int i = 1; i < *argc; i++) { char *arg1; arg1 = argv[i]; if (strlen(arg1) > 1 && arg1[0] == '+') { int c1, f1; char sopts1[128]; sopts1[0] = 0; c1 = f1 = 0; if (parse_pos_obs(arg1 + 1, &f1, &c1, sopts1) < 0) continue; else { f1 += 1; c1 += 1; if (i + 1 < *argc) { char *arg2 = argv[i + 1]; if (strlen(arg2) > 1 && arg2[0] == '-') { int c2, f2; char sopts2[128]; sopts2[0] = 0; c2 = f2 = 0; if (parse_pos_obs(arg2 + 1, &f2, &c2, sopts2) >= 0) { if (c2 > 0) f2 += 1; sprintf(sopt, "-k%d.%d%s,%d.%d%s", f1, c1, sopts1, f2, c2, sopts2); argv[i] = sort_strdup(sopt); for (int j = i + 1; j + 1 < *argc; j++) argv[j] = argv[j + 1]; *argc -= 1; continue; } } } sprintf(sopt, "-k%d.%d%s", f1, c1, sopts1); argv[i] = sort_strdup(sopt); } } } } /* * Seed random sort */ static void get_random_seed(const char *random_source) { char randseed[32]; struct stat fsb, rsb; ssize_t rd; int rsfd; rsfd = -1; rd = sizeof(randseed); if (random_source == NULL) { if (getentropy(randseed, sizeof(randseed)) < 0) err(EX_SOFTWARE, "getentropy"); goto out; } rsfd = open(random_source, O_RDONLY | O_CLOEXEC); if (rsfd < 0) err(EX_NOINPUT, "open: %s", random_source); if (fstat(rsfd, &fsb) != 0) err(EX_SOFTWARE, "fstat"); if (!S_ISREG(fsb.st_mode) && !S_ISCHR(fsb.st_mode)) err(EX_USAGE, "random seed isn't a regular file or /dev/random"); /* * Regular files: read up to maximum seed size and explicitly * reject longer files. */ if (S_ISREG(fsb.st_mode)) { if (fsb.st_size > (off_t)sizeof(randseed)) errx(EX_USAGE, "random seed is too large (%jd >" " %zu)!", (intmax_t)fsb.st_size, sizeof(randseed)); else if (fsb.st_size < 1) errx(EX_USAGE, "random seed is too small (" "0 bytes)"); memset(randseed, 0, sizeof(randseed)); rd = read(rsfd, randseed, fsb.st_size); if (rd < 0) err(EX_SOFTWARE, "reading random seed file %s", random_source); if (rd < (ssize_t)fsb.st_size) errx(EX_SOFTWARE, "short read from %s", random_source); } else if (S_ISCHR(fsb.st_mode)) { if (stat("/dev/random", &rsb) < 0) err(EX_SOFTWARE, "stat"); if (fsb.st_dev != rsb.st_dev || fsb.st_ino != rsb.st_ino) errx(EX_USAGE, "random seed is a character " "device other than /dev/random"); if (getentropy(randseed, sizeof(randseed)) < 0) err(EX_SOFTWARE, "getentropy"); } out: if (rsfd >= 0) close(rsfd); MD5Init(&md5_ctx); MD5Update(&md5_ctx, randseed, rd); } /* * Main function. */ int main(int argc, char **argv) { char *outfile, *real_outfile; char *random_source = NULL; int c, result; bool mef_flags[NUMBER_OF_MUTUALLY_EXCLUSIVE_FLAGS] = { false, false, false, false, false, false }; result = 0; outfile = sort_strdup("-"); real_outfile = NULL; struct sort_mods *sm = &default_sort_mods_object; init_tmp_files(); set_signal_handler(); set_hw_params(); set_locale(); set_tmpdir(); set_sort_opts(); fix_obsolete_keys(&argc, argv); while (((c = getopt_long(argc, argv, OPTIONS, long_options, NULL)) != -1)) { check_mutually_exclusive_flags(c, mef_flags); if (!set_sort_modifier(sm, c)) { switch (c) { case 'c': sort_opts_vals.cflag = true; if (optarg) { if (!strcmp(optarg, "diagnose-first")) ; else if (!strcmp(optarg, "silent") || !strcmp(optarg, "quiet")) sort_opts_vals.csilentflag = true; else if (*optarg) unknown(optarg); } break; case 'C': sort_opts_vals.cflag = true; sort_opts_vals.csilentflag = true; break; case 'k': { sort_opts_vals.complex_sort = true; sort_opts_vals.kflag = true; keys_num++; keys = sort_realloc(keys, keys_num * sizeof(struct key_specs)); memset(&(keys[keys_num - 1]), 0, sizeof(struct key_specs)); if (parse_k(optarg, &(keys[keys_num - 1])) < 0) { errc(2, EINVAL, "-k %s", optarg); } break; } case 'm': sort_opts_vals.mflag = true; break; case 'o': outfile = sort_realloc(outfile, (strlen(optarg) + 1)); strcpy(outfile, optarg); break; case 's': sort_opts_vals.sflag = true; break; case 'S': available_free_memory = parse_memory_buffer_value(optarg); break; case 'T': tmpdir = sort_strdup(optarg); break; case 't': while (strlen(optarg) > 1) { if (optarg[0] != '\\') { errc(2, EINVAL, "%s", optarg); } optarg += 1; if (*optarg == '0') { *optarg = 0; break; } } sort_opts_vals.tflag = true; sort_opts_vals.field_sep = btowc(optarg[0]); if (sort_opts_vals.field_sep == WEOF) { errno = EINVAL; err(2, NULL); } if (!gnusort_numeric_compatibility) { if (symbol_decimal_point == sort_opts_vals.field_sep) symbol_decimal_point = WEOF; if (symbol_thousands_sep == sort_opts_vals.field_sep) symbol_thousands_sep = WEOF; if (symbol_negative_sign == sort_opts_vals.field_sep) symbol_negative_sign = WEOF; if (symbol_positive_sign == sort_opts_vals.field_sep) symbol_positive_sign = WEOF; } break; case 'u': sort_opts_vals.uflag = true; /* stable sort for the correct unique val */ sort_opts_vals.sflag = true; break; case 'z': sort_opts_vals.zflag = true; break; case SORT_OPT: if (optarg) { if (!strcmp(optarg, "general-numeric")) set_sort_modifier(sm, 'g'); else if (!strcmp(optarg, "human-numeric")) set_sort_modifier(sm, 'h'); else if (!strcmp(optarg, "numeric")) set_sort_modifier(sm, 'n'); else if (!strcmp(optarg, "month")) set_sort_modifier(sm, 'M'); else if (!strcmp(optarg, "random")) set_sort_modifier(sm, 'R'); else unknown(optarg); } break; #if defined(SORT_THREADS) case PARALLEL_OPT: nthreads = (size_t)(atoi(optarg)); if (nthreads < 1) nthreads = 1; if (nthreads > 1024) nthreads = 1024; break; #endif case QSORT_OPT: sort_opts_vals.sort_method = SORT_QSORT; break; case MERGESORT_OPT: sort_opts_vals.sort_method = SORT_MERGESORT; break; case MMAP_OPT: use_mmap = true; break; case HEAPSORT_OPT: sort_opts_vals.sort_method = SORT_HEAPSORT; break; case RADIXSORT_OPT: sort_opts_vals.sort_method = SORT_RADIXSORT; break; case RANDOMSOURCE_OPT: random_source = strdup(optarg); break; case COMPRESSPROGRAM_OPT: compress_program = strdup(optarg); break; case FF_OPT: read_fns_from_file0(optarg); break; case BS_OPT: { errno = 0; long mof = strtol(optarg, NULL, 10); if (errno != 0) err(2, "--batch-size"); if (mof >= 2) max_open_files = (size_t) mof + 1; } break; case VERSION_OPT: printf("%s\n", VERSION); exit(EXIT_SUCCESS); /* NOTREACHED */ break; case DEBUG_OPT: debug_sort = true; break; case HELP_OPT: usage(false); /* NOTREACHED */ break; default: usage(true); /* NOTREACHED */ } } } argc -= optind; argv += optind; if (argv_from_file0) { argc = argc_from_file0; argv = argv_from_file0; } #ifndef WITHOUT_NLS catalog = catopen("sort", NL_CAT_LOCALE); #endif if (sort_opts_vals.cflag && sort_opts_vals.mflag) errx(1, "%c:%c: %s", 'm', 'c', getstr(1)); #ifndef WITHOUT_NLS catclose(catalog); #endif if (keys_num == 0) { keys_num = 1; keys = sort_realloc(keys, sizeof(struct key_specs)); memset(&(keys[0]), 0, sizeof(struct key_specs)); keys[0].c1 = 1; keys[0].pos1b = default_sort_mods->bflag; keys[0].pos2b = default_sort_mods->bflag; memcpy(&(keys[0].sm), default_sort_mods, sizeof(struct sort_mods)); } for (size_t i = 0; i < keys_num; i++) { struct key_specs *ks; ks = &(keys[i]); if (sort_modifier_empty(&(ks->sm)) && !(ks->pos1b) && !(ks->pos2b)) { ks->pos1b = sm->bflag; ks->pos2b = sm->bflag; memcpy(&(ks->sm), sm, sizeof(struct sort_mods)); } ks->sm.func = get_sort_func(&(ks->sm)); } if (debug_sort) { printf("Memory to be used for sorting: %llu\n",available_free_memory); #if defined(SORT_THREADS) printf("Number of CPUs: %d\n",(int)ncpu); nthreads = 1; #endif printf("Using collate rules of %s locale\n", setlocale(LC_COLLATE, NULL)); if (byte_sort) printf("Byte sort is used\n"); if (print_symbols_on_debug) { printf("Decimal Point: <%lc>\n", symbol_decimal_point); if (symbol_thousands_sep) printf("Thousands separator: <%lc>\n", symbol_thousands_sep); printf("Positive sign: <%lc>\n", symbol_positive_sign); printf("Negative sign: <%lc>\n", symbol_negative_sign); } } if (need_random) get_random_seed(random_source); /* Case when the outfile equals one of the input files: */ if (strcmp(outfile, "-")) { for(int i = 0; i < argc; ++i) { if (strcmp(argv[i], outfile) == 0) { real_outfile = sort_strdup(outfile); for(;;) { char* tmp = sort_malloc(strlen(outfile) + strlen(".tmp") + 1); strcpy(tmp, outfile); strcpy(tmp + strlen(tmp), ".tmp"); sort_free(outfile); outfile = tmp; if (access(outfile, F_OK) < 0) break; } tmp_file_atexit(outfile); } } } #if defined(SORT_THREADS) if ((argc < 1) || (strcmp(outfile, "-") == 0) || (*outfile == 0)) nthreads = 1; #endif if (!sort_opts_vals.cflag && !sort_opts_vals.mflag) { struct file_list fl; struct sort_list list; sort_list_init(&list); file_list_init(&fl, true); if (argc < 1) procfile("-", &list, &fl); else { while (argc > 0) { procfile(*argv, &list, &fl); --argc; ++argv; } } if (fl.count < 1) sort_list_to_file(&list, outfile); else { if (list.count > 0) { char *flast = new_tmp_file_name(); sort_list_to_file(&list, flast); file_list_add(&fl, flast, false); } merge_files(&fl, outfile); } file_list_clean(&fl); /* * We are about to exit the program, so we can ignore * the clean-up for speed * * sort_list_clean(&list); */ } else if (sort_opts_vals.cflag) { result = (argc == 0) ? (check("-")) : (check(*argv)); } else if (sort_opts_vals.mflag) { struct file_list fl; file_list_init(&fl, false); /* No file arguments remaining means "read from stdin." */ if (argc == 0) file_list_add(&fl, "-", true); else file_list_populate(&fl, argc, argv, true); merge_files(&fl, outfile); file_list_clean(&fl); } if (real_outfile) { unlink(real_outfile); if (rename(outfile, real_outfile) < 0) err(2, NULL); sort_free(real_outfile); } sort_free(outfile); return (result); }