Index: head/usr.bin/localedef/collate.c =================================================================== --- head/usr.bin/localedef/collate.c (revision 290558) +++ head/usr.bin/localedef/collate.c (revision 290559) @@ -1,1315 +1,1310 @@ /* * Copyright 2010 Nexenta Systems, Inc. All rights reserved. * Copyright 2015 John Marino * * This source code is derived from the illumos localedef command, and * provided under BSD-style license terms by Nexenta Systems, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * LC_COLLATE database generation routines for localedef. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include "localedef.h" #include "parser.h" #include "collate.h" /* * Design notes. * * It will be extremely helpful to the reader if they have access to * the localedef and locale file format specifications available. * Latest versions of these are available from www.opengroup.org. * * The design for the collation code is a bit complex. The goal is a * single collation database as described in collate.h (in * libc/port/locale). However, there are some other tidbits: * * a) The substitution entries are now a directly indexable array. A * priority elsewhere in the table is taken as an index into the * substitution table if it has a high bit (COLLATE_SUBST_PRIORITY) * set. (The bit is cleared and the result is the index into the * table. * * b) We eliminate duplicate entries into the substitution table. * This saves a lot of space. * * c) The priorities for each level are "compressed", so that each * sorting level has consecutively numbered priorities starting at 1. * (O is reserved for the ignore priority.) This means sort levels * which only have a few distinct priorities can represent the * priority level in fewer bits, which makes the strxfrm output * smaller. * * d) We record the total number of priorities so that strxfrm can * figure out how many bytes to expand a numeric priority into. * * e) For the UNDEFINED pass (the last pass), we record the maximum * number of bits needed to uniquely prioritize these entries, so that * the last pass can also use smaller strxfrm output when possible. * * f) Priorities with the sign bit set are verboten. This works out * because no active character set needs that bit to carry significant * information once the character is in wide form. * * To process the entire data to make the database, we actually run * multiple passes over the data. * * The first pass, which is done at parse time, identifies elements, * substitutions, and such, and records them in priority order. As * some priorities can refer to other priorities, using forward * references, we use a table of references indicating whether the * priority's value has been resolved, or whether it is still a * reference. * * The second pass walks over all the items in priority order, noting * that they are used directly, and not just an indirect reference. * This is done by creating a "weight" structure for the item. The * weights are stashed in an RB tree sorted by relative "priority". * * The third pass walks over all the weight structures, in priority * order, and assigns a new monotonically increasing (per sort level) * weight value to them. These are the values that will actually be * written to the file. * * The fourth pass just writes the data out. */ /* * In order to resolve the priorities, we create a table of priorities. * Entries in the table can be in one of three states. * * UNKNOWN is for newly allocated entries, and indicates that nothing * is known about the priority. (For example, when new entries are created * for collating-symbols, this is the value assigned for them until the * collating symbol's order has been determined. * * RESOLVED is used for an entry where the priority indicates the final * numeric weight. * * REFER is used for entries that reference other entries. Typically * this is used for forward references. A collating-symbol can never * have this value. * * The "pass" field is used during final resolution to aid in detection * of referencing loops. (For example depends on , but has its * priority dependent on .) */ typedef enum { UNKNOWN, /* priority is totally unknown */ RESOLVED, /* priority value fully resolved */ REFER /* priority is a reference (index) */ } res_t; typedef struct weight { int32_t pri; int opt; RB_ENTRY(weight) entry; } weight_t; typedef struct priority { res_t res; int32_t pri; int pass; int lineno; } collpri_t; #define NUM_WT collinfo.directive_count /* * These are the abstract collating symbols, which are just a symbolic * way to reference a priority. */ struct collsym { char *name; int32_t ref; RB_ENTRY(collsym) entry; }; /* * These are also abstract collating symbols, but we allow them to have * different priorities at different levels. */ typedef struct collundef { char *name; int32_t ref[COLL_WEIGHTS_MAX]; RB_ENTRY(collundef) entry; } collundef_t; /* * These are called "chains" in libc. This records the fact that two * more characters should be treated as a single collating entity when * they appear together. For example, in Spanish gets collated * as a character between and . */ struct collelem { char *symbol; wchar_t *expand; int32_t ref[COLL_WEIGHTS_MAX]; RB_ENTRY(collelem) rb_bysymbol; RB_ENTRY(collelem) rb_byexpand; }; /* * Individual characters have a sequence of weights as well. */ typedef struct collchar { wchar_t wc; int32_t ref[COLL_WEIGHTS_MAX]; RB_ENTRY(collchar) entry; } collchar_t; /* * Substitution entries. The key is itself a priority. Note that * when we create one of these, we *automatically* wind up with a * fully resolved priority for the key, because creation of * substitutions creates a resolved priority at the same time. */ typedef struct subst{ int32_t key; int32_t ref[COLLATE_STR_LEN]; RB_ENTRY(subst) entry; RB_ENTRY(subst) entry_ref; } subst_t; static RB_HEAD(collsyms, collsym) collsyms; static RB_HEAD(collundefs, collundef) collundefs; static RB_HEAD(elem_by_symbol, collelem) elem_by_symbol; static RB_HEAD(elem_by_expand, collelem) elem_by_expand; static RB_HEAD(collchars, collchar) collchars; static RB_HEAD(substs, subst) substs[COLL_WEIGHTS_MAX]; static RB_HEAD(substs_ref, subst) substs_ref[COLL_WEIGHTS_MAX]; static RB_HEAD(weights, weight) weights[COLL_WEIGHTS_MAX]; static int32_t nweight[COLL_WEIGHTS_MAX]; /* * This is state tracking for the ellipsis token. Note that we start * the initial values so that the ellipsis logic will think we got a * magic starting value of NUL. It starts at minus one because the * starting point is exclusive -- i.e. the starting point is not * itself handled by the ellipsis code. */ static int currorder = EOF; static int lastorder = EOF; static collelem_t *currelem; static collchar_t *currchar; static collundef_t *currundef; static wchar_t ellipsis_start = 0; static int32_t ellipsis_weights[COLL_WEIGHTS_MAX]; /* * We keep a running tally of weights. */ static int nextpri = 1; static int nextsubst[COLL_WEIGHTS_MAX] = { 0 }; /* * This array collects up the weights for each level. */ static int32_t order_weights[COLL_WEIGHTS_MAX]; static int curr_weight = 0; static int32_t subst_weights[COLLATE_STR_LEN]; static int curr_subst = 0; /* * Some initial priority values. */ static int32_t pri_undefined[COLL_WEIGHTS_MAX]; static int32_t pri_ignore; static collate_info_t collinfo; static collpri_t *prilist = NULL; static int numpri = 0; static int maxpri = 0; static void start_order(int); static int32_t new_pri(void) { int i; if (numpri >= maxpri) { maxpri = maxpri ? maxpri * 2 : 1024; prilist = realloc(prilist, sizeof (collpri_t) * maxpri); if (prilist == NULL) { fprintf(stderr,"out of memory"); return (-1); } for (i = numpri; i < maxpri; i++) { prilist[i].res = UNKNOWN; prilist[i].pri = 0; prilist[i].pass = 0; } } return (numpri++); } static collpri_t * get_pri(int32_t ref) { if ((ref < 0) || (ref > numpri)) { INTERR; return (NULL); } return (&prilist[ref]); } static void set_pri(int32_t ref, int32_t v, res_t res) { collpri_t *pri; pri = get_pri(ref); if ((res == REFER) && ((v < 0) || (v >= numpri))) { INTERR; } /* Resolve self references */ if ((res == REFER) && (ref == v)) { v = nextpri; res = RESOLVED; } if (pri->res != UNKNOWN) { warn("repeated item in order list (first on %d)", pri->lineno); return; } pri->lineno = lineno; pri->pri = v; pri->res = res; } static int32_t resolve_pri(int32_t ref) { collpri_t *pri; static int32_t pass = 0; pri = get_pri(ref); pass++; while (pri->res == REFER) { if (pri->pass == pass) { /* report a line with the circular symbol */ lineno = pri->lineno; fprintf(stderr,"circular reference in order list"); return (-1); } if ((pri->pri < 0) || (pri->pri >= numpri)) { INTERR; return (-1); } pri->pass = pass; pri = &prilist[pri->pri]; } if (pri->res == UNKNOWN) { return (-1); } if (pri->res != RESOLVED) INTERR; return (pri->pri); } static int weight_compare(const void *n1, const void *n2) { int32_t k1 = ((const weight_t *)n1)->pri; int32_t k2 = ((const weight_t *)n2)->pri; return (k1 < k2 ? -1 : k1 > k2 ? 1 : 0); } RB_GENERATE_STATIC(weights, weight, entry, weight_compare); static int collsym_compare(const void *n1, const void *n2) { const collsym_t *c1 = n1; const collsym_t *c2 = n2; int rv; rv = strcmp(c1->name, c2->name); return ((rv < 0) ? -1 : (rv > 0) ? 1 : 0); } RB_GENERATE_STATIC(collsyms, collsym, entry, collsym_compare); static int collundef_compare(const void *n1, const void *n2) { const collundef_t *c1 = n1; const collundef_t *c2 = n2; int rv; rv = strcmp(c1->name, c2->name); return ((rv < 0) ? -1 : (rv > 0) ? 1 : 0); } RB_GENERATE_STATIC(collundefs, collundef, entry, collundef_compare); static int element_compare_symbol(const void *n1, const void *n2) { const collelem_t *c1 = n1; const collelem_t *c2 = n2; int rv; rv = strcmp(c1->symbol, c2->symbol); return ((rv < 0) ? -1 : (rv > 0) ? 1 : 0); } RB_GENERATE_STATIC(elem_by_symbol, collelem, rb_bysymbol, element_compare_symbol); static int element_compare_expand(const void *n1, const void *n2) { const collelem_t *c1 = n1; const collelem_t *c2 = n2; int rv; rv = wcscmp(c1->expand, c2->expand); return ((rv < 0) ? -1 : (rv > 0) ? 1 : 0); } RB_GENERATE_STATIC(elem_by_expand, collelem, rb_byexpand, element_compare_expand); static int collchar_compare(const void *n1, const void *n2) { wchar_t k1 = ((const collchar_t *)n1)->wc; wchar_t k2 = ((const collchar_t *)n2)->wc; return (k1 < k2 ? -1 : k1 > k2 ? 1 : 0); } RB_GENERATE_STATIC(collchars, collchar, entry, collchar_compare); static int subst_compare(const void *n1, const void *n2) { int32_t k1 = ((const subst_t *)n1)->key; int32_t k2 = ((const subst_t *)n2)->key; return (k1 < k2 ? -1 : k1 > k2 ? 1 : 0); } RB_GENERATE_STATIC(substs, subst, entry, subst_compare); -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wcast-qual" - static int subst_compare_ref(const void *n1, const void *n2) { - int32_t *c1 = ((subst_t *)n1)->ref; - int32_t *c2 = ((subst_t *)n2)->ref; + const wchar_t *c1 = ((const subst_t *)n1)->ref; + const wchar_t *c2 = ((const subst_t *)n2)->ref; int rv; - rv = wcscmp((wchar_t *)c1, (wchar_t *)c2); + rv = wcscmp(c1, c2); return ((rv < 0) ? -1 : (rv > 0) ? 1 : 0); } RB_GENERATE_STATIC(substs_ref, subst, entry_ref, subst_compare_ref); - -#pragma GCC diagnostic pop void init_collate(void) { int i; RB_INIT(&collsyms); RB_INIT(&collundefs); RB_INIT(&elem_by_symbol); RB_INIT(&elem_by_expand); RB_INIT(&collchars); for (i = 0; i < COLL_WEIGHTS_MAX; i++) { RB_INIT(&substs[i]); RB_INIT(&substs_ref[i]); RB_INIT(&weights[i]); nweight[i] = 1; } (void) memset(&collinfo, 0, sizeof (collinfo)); /* allocate some initial priorities */ pri_ignore = new_pri(); set_pri(pri_ignore, 0, RESOLVED); for (i = 0; i < COLL_WEIGHTS_MAX; i++) { pri_undefined[i] = new_pri(); /* we will override this later */ set_pri(pri_undefined[i], COLLATE_MAX_PRIORITY, UNKNOWN); } } void define_collsym(char *name) { collsym_t *sym; if ((sym = calloc(sizeof (*sym), 1)) == NULL) { fprintf(stderr,"out of memory"); return; } sym->name = name; sym->ref = new_pri(); if (RB_FIND(collsyms, &collsyms, sym) != NULL) { /* * This should never happen because we are only called * for undefined symbols. */ INTERR; return; } RB_INSERT(collsyms, &collsyms, sym); } collsym_t * lookup_collsym(char *name) { collsym_t srch; srch.name = name; return (RB_FIND(collsyms, &collsyms, &srch)); } collelem_t * lookup_collelem(char *symbol) { collelem_t srch; srch.symbol = symbol; return (RB_FIND(elem_by_symbol, &elem_by_symbol, &srch)); } static collundef_t * get_collundef(char *name) { collundef_t srch; collundef_t *ud; int i; srch.name = name; if ((ud = RB_FIND(collundefs, &collundefs, &srch)) == NULL) { if (((ud = calloc(sizeof (*ud), 1)) == NULL) || ((ud->name = strdup(name)) == NULL)) { fprintf(stderr,"out of memory"); return (NULL); } for (i = 0; i < NUM_WT; i++) { ud->ref[i] = new_pri(); } RB_INSERT(collundefs, &collundefs, ud); } add_charmap_undefined(name); return (ud); } static collchar_t * get_collchar(wchar_t wc, int create) { collchar_t srch; collchar_t *cc; int i; srch.wc = wc; cc = RB_FIND(collchars, &collchars, &srch); if ((cc == NULL) && create) { if ((cc = calloc(sizeof (*cc), 1)) == NULL) { fprintf(stderr, "out of memory"); return (NULL); } for (i = 0; i < NUM_WT; i++) { cc->ref[i] = new_pri(); } cc->wc = wc; RB_INSERT(collchars, &collchars, cc); } return (cc); } void end_order_collsym(collsym_t *sym) { start_order(T_COLLSYM); /* update the weight */ set_pri(sym->ref, nextpri, RESOLVED); nextpri++; } void end_order(void) { int i; int32_t pri; int32_t ref; collpri_t *p; /* advance the priority/weight */ pri = nextpri; switch (currorder) { case T_CHAR: for (i = 0; i < NUM_WT; i++) { if (((ref = order_weights[i]) < 0) || ((p = get_pri(ref)) == NULL) || (p->pri == -1)) { /* unspecified weight is a self reference */ set_pri(currchar->ref[i], pri, RESOLVED); } else { set_pri(currchar->ref[i], ref, REFER); } order_weights[i] = -1; } /* leave a cookie trail in case next symbol is ellipsis */ ellipsis_start = currchar->wc + 1; currchar = NULL; break; case T_ELLIPSIS: /* save off the weights were we can find them */ for (i = 0; i < NUM_WT; i++) { ellipsis_weights[i] = order_weights[i]; order_weights[i] = -1; } break; case T_COLLELEM: if (currelem == NULL) { INTERR; } else { for (i = 0; i < NUM_WT; i++) { if (((ref = order_weights[i]) < 0) || ((p = get_pri(ref)) == NULL) || (p->pri == -1)) { set_pri(currelem->ref[i], pri, RESOLVED); } else { set_pri(currelem->ref[i], ref, REFER); } order_weights[i] = -1; } } break; case T_UNDEFINED: for (i = 0; i < NUM_WT; i++) { if (((ref = order_weights[i]) < 0) || ((p = get_pri(ref)) == NULL) || (p->pri == -1)) { set_pri(pri_undefined[i], -1, RESOLVED); } else { set_pri(pri_undefined[i], ref, REFER); } order_weights[i] = -1; } break; case T_SYMBOL: for (i = 0; i < NUM_WT; i++) { if (((ref = order_weights[i]) < 0) || ((p = get_pri(ref)) == NULL) || (p->pri == -1)) { set_pri(currundef->ref[i], pri, RESOLVED); } else { set_pri(currundef->ref[i], ref, REFER); } order_weights[i] = -1; } break; default: INTERR; } nextpri++; } static void start_order(int type) { int i; lastorder = currorder; currorder = type; /* this is used to protect ELLIPSIS processing */ if ((lastorder == T_ELLIPSIS) && (type != T_CHAR)) { fprintf(stderr, "character value expected"); } for (i = 0; i < COLL_WEIGHTS_MAX; i++) { order_weights[i] = -1; } curr_weight = 0; } void start_order_undefined(void) { start_order(T_UNDEFINED); } void start_order_symbol(char *name) { currundef = get_collundef(name); start_order(T_SYMBOL); } void start_order_char(wchar_t wc) { collchar_t *cc; int32_t ref; start_order(T_CHAR); /* * If we last saw an ellipsis, then we need to close the range. * Handle that here. Note that we have to be careful because the * items *inside* the range are treated exclusiveley to the items * outside of the range. The ends of the range can have quite * different weights than the range members. */ if (lastorder == T_ELLIPSIS) { int i; if (wc < ellipsis_start) { fprintf(stderr, "malformed range!"); return; } while (ellipsis_start < wc) { /* * pick all of the saved weights for the * ellipsis. note that -1 encodes for the * ellipsis itself, which means to take the * current relative priority. */ if ((cc = get_collchar(ellipsis_start, 1)) == NULL) { INTERR; return; } for (i = 0; i < NUM_WT; i++) { collpri_t *p; if (((ref = ellipsis_weights[i]) == -1) || ((p = get_pri(ref)) == NULL) || (p->pri == -1)) { set_pri(cc->ref[i], nextpri, RESOLVED); } else { set_pri(cc->ref[i], ref, REFER); } ellipsis_weights[i] = 0; } ellipsis_start++; nextpri++; } } currchar = get_collchar(wc, 1); } void start_order_collelem(collelem_t *e) { start_order(T_COLLELEM); currelem = e; } void start_order_ellipsis(void) { int i; start_order(T_ELLIPSIS); if (lastorder != T_CHAR) { fprintf(stderr, "illegal starting point for range"); return; } for (i = 0; i < NUM_WT; i++) { ellipsis_weights[i] = order_weights[i]; } } void define_collelem(char *name, wchar_t *wcs) { collelem_t *e; int i; if (wcslen(wcs) >= COLLATE_STR_LEN) { fprintf(stderr,"expanded collation element too long"); return; } if ((e = calloc(sizeof (*e), 1)) == NULL) { fprintf(stderr, "out of memory"); return; } e->expand = wcs; e->symbol = name; /* * This is executed before the order statement, so we don't * know how many priorities we *really* need. We allocate one * for each possible weight. Not a big deal, as collating-elements * prove to be quite rare. */ for (i = 0; i < COLL_WEIGHTS_MAX; i++) { e->ref[i] = new_pri(); } /* A character sequence can only reduce to one element. */ if ((RB_FIND(elem_by_symbol, &elem_by_symbol, e) != NULL) || (RB_FIND(elem_by_expand, &elem_by_expand, e) != NULL)) { fprintf(stderr, "duplicate collating element definition"); return; } RB_INSERT(elem_by_symbol, &elem_by_symbol, e); RB_INSERT(elem_by_expand, &elem_by_expand, e); } void add_order_bit(int kw) { uint8_t bit = DIRECTIVE_UNDEF; switch (kw) { case T_FORWARD: bit = DIRECTIVE_FORWARD; break; case T_BACKWARD: bit = DIRECTIVE_BACKWARD; break; case T_POSITION: bit = DIRECTIVE_POSITION; break; default: INTERR; break; } collinfo.directive[collinfo.directive_count] |= bit; } void add_order_directive(void) { if (collinfo.directive_count >= COLL_WEIGHTS_MAX) { fprintf(stderr,"too many directives (max %d)", COLL_WEIGHTS_MAX); } collinfo.directive_count++; } static void add_order_pri(int32_t ref) { if (curr_weight >= NUM_WT) { fprintf(stderr,"too many weights (max %d)", NUM_WT); return; } order_weights[curr_weight] = ref; curr_weight++; } void add_order_collsym(collsym_t *s) { add_order_pri(s->ref); } void add_order_char(wchar_t wc) { collchar_t *cc; if ((cc = get_collchar(wc, 1)) == NULL) { INTERR; return; } add_order_pri(cc->ref[curr_weight]); } void add_order_collelem(collelem_t *e) { add_order_pri(e->ref[curr_weight]); } void add_order_ignore(void) { add_order_pri(pri_ignore); } void add_order_symbol(char *sym) { collundef_t *c; if ((c = get_collundef(sym)) == NULL) { INTERR; return; } add_order_pri(c->ref[curr_weight]); } void add_order_ellipsis(void) { /* special NULL value indicates self reference */ add_order_pri(0); } void add_order_subst(void) { subst_t srch; subst_t *s; int i; (void) memset(&srch, 0, sizeof (srch)); for (i = 0; i < curr_subst; i++) { srch.ref[i] = subst_weights[i]; subst_weights[i] = 0; } s = RB_FIND(substs_ref, &substs_ref[curr_weight], &srch); if (s == NULL) { if ((s = calloc(sizeof (*s), 1)) == NULL) { fprintf(stderr,"out of memory"); return; } s->key = new_pri(); /* * We use a self reference for our key, but we set a * high bit to indicate that this is a substitution * reference. This will expedite table lookups later, * and prevent table lookups for situations that don't * require it. (In short, its a big win, because we * can skip a lot of binary searching.) */ set_pri(s->key, (nextsubst[curr_weight] | COLLATE_SUBST_PRIORITY), RESOLVED); nextsubst[curr_weight] += 1; for (i = 0; i < curr_subst; i++) { s->ref[i] = srch.ref[i]; } RB_INSERT(substs_ref, &substs_ref[curr_weight], s); if (RB_FIND(substs, &substs[curr_weight], s) != NULL) { INTERR; return; } RB_INSERT(substs, &substs[curr_weight], s); } curr_subst = 0; /* * We are using the current (unique) priority as a search key * in the substitution table. */ add_order_pri(s->key); } static void add_subst_pri(int32_t ref) { if (curr_subst >= COLLATE_STR_LEN) { fprintf(stderr,"substitution string is too long"); return; } subst_weights[curr_subst] = ref; curr_subst++; } void add_subst_char(wchar_t wc) { collchar_t *cc; if (((cc = get_collchar(wc, 1)) == NULL) || (cc->wc != wc)) { INTERR; return; } /* we take the weight for the character at that position */ add_subst_pri(cc->ref[curr_weight]); } void add_subst_collelem(collelem_t *e) { add_subst_pri(e->ref[curr_weight]); } void add_subst_collsym(collsym_t *s) { add_subst_pri(s->ref); } void add_subst_symbol(char *ptr) { collundef_t *cu; if ((cu = get_collundef(ptr)) != NULL) { add_subst_pri(cu->ref[curr_weight]); } } void add_weight(int32_t ref, int pass) { weight_t srch; weight_t *w; srch.pri = resolve_pri(ref); /* No translation of ignores */ if (srch.pri == 0) return; /* Substitution priorities are not weights */ if (srch.pri & COLLATE_SUBST_PRIORITY) return; if (RB_FIND(weights, &weights[pass], &srch) != NULL) return; if ((w = calloc(sizeof (*w), 1)) == NULL) { fprintf(stderr, "out of memory"); return; } w->pri = srch.pri; RB_INSERT(weights, &weights[pass], w); } void add_weights(int32_t *refs) { int i; for (i = 0; i < NUM_WT; i++) { add_weight(refs[i], i); } } int32_t get_weight(int32_t ref, int pass) { weight_t srch; weight_t *w; int32_t pri; pri = resolve_pri(ref); if (pri & COLLATE_SUBST_PRIORITY) { return (pri); } if (pri <= 0) { return (pri); } srch.pri = pri; if ((w = RB_FIND(weights, &weights[pass], &srch)) == NULL) { INTERR; return (-1); } return (w->opt); } wchar_t * wsncpy(wchar_t *s1, const wchar_t *s2, size_t n) { wchar_t *os1 = s1; n++; while (--n > 0 && (*s1++ = *s2++) != 0) continue; if (n > 0) while (--n > 0) *s1++ = 0; return (os1); } #define RB_COUNT(x, name, head, cnt) do { \ (cnt) = 0; \ RB_FOREACH(x, name, (head)) { \ (cnt)++; \ } \ } while (0) #define RB_NUMNODES(type, name, head, cnt) do { \ type *t; \ cnt = 0; \ RB_FOREACH(t, name, head) { \ cnt++; \ } \ } while (0) void dump_collate(void) { FILE *f; int i, j, n; size_t sz; int32_t pri; collelem_t *ce; collchar_t *cc; subst_t *sb; char vers[COLLATE_STR_LEN]; collate_char_t chars[UCHAR_MAX + 1]; collate_large_t *large; collate_subst_t *subst[COLL_WEIGHTS_MAX]; collate_chain_t *chain; /* * We have to run throught a preliminary pass to identify all the * weights that we use for each sorting level. */ for (i = 0; i < NUM_WT; i++) { add_weight(pri_ignore, i); } for (i = 0; i < NUM_WT; i++) { RB_FOREACH(sb, substs, &substs[i]) { for (j = 0; sb->ref[j]; j++) { add_weight(sb->ref[j], i); } } } RB_FOREACH(ce, elem_by_expand, &elem_by_expand) { add_weights(ce->ref); } RB_FOREACH(cc, collchars, &collchars) { add_weights(cc->ref); } /* * Now we walk the entire set of weights, removing the gaps * in the weights. This gives us optimum usage. The walk * occurs in priority. */ for (i = 0; i < NUM_WT; i++) { weight_t *w; RB_FOREACH(w, weights, &weights[i]) { w->opt = nweight[i]; nweight[i] += 1; } } (void) memset(&chars, 0, sizeof (chars)); (void) memset(vers, 0, COLLATE_STR_LEN); (void) strlcpy(vers, COLLATE_VERSION, sizeof (vers)); /* * We need to make sure we arrange for the UNDEFINED field * to show up. Also, set the total weight counts. */ for (i = 0; i < NUM_WT; i++) { if (resolve_pri(pri_undefined[i]) == -1) { set_pri(pri_undefined[i], -1, RESOLVED); /* they collate at the end of everything else */ collinfo.undef_pri[i] = COLLATE_MAX_PRIORITY; } collinfo.pri_count[i] = nweight[i]; } collinfo.pri_count[NUM_WT] = max_wide(); collinfo.undef_pri[NUM_WT] = COLLATE_MAX_PRIORITY; collinfo.directive[NUM_WT] = DIRECTIVE_UNDEFINED; /* * Ordinary character priorities */ for (i = 0; i <= UCHAR_MAX; i++) { if ((cc = get_collchar(i, 0)) != NULL) { for (j = 0; j < NUM_WT; j++) { chars[i].pri[j] = get_weight(cc->ref[j], j); } } else { for (j = 0; j < NUM_WT; j++) { chars[i].pri[j] = get_weight(pri_undefined[j], j); } /* * Per POSIX, for undefined characters, we * also have to add a last item, which is the * character code. */ chars[i].pri[NUM_WT] = i; } } /* * Substitution tables */ for (i = 0; i < NUM_WT; i++) { collate_subst_t *st = NULL; subst_t *temp; RB_COUNT(temp, substs, &substs[i], n); collinfo.subst_count[i] = n; if ((st = calloc(sizeof (collate_subst_t) * n, 1)) == NULL) { fprintf(stderr, "out of memory"); return; } n = 0; RB_FOREACH(sb, substs, &substs[i]) { if ((st[n].key = resolve_pri(sb->key)) < 0) { /* by definition these resolve! */ INTERR; } if (st[n].key != (n | COLLATE_SUBST_PRIORITY)) { INTERR; } for (j = 0; sb->ref[j]; j++) { st[n].pri[j] = get_weight(sb->ref[j], i); } n++; } if (n != collinfo.subst_count[i]) INTERR; subst[i] = st; } /* * Chains, i.e. collating elements */ RB_NUMNODES(collelem_t, elem_by_expand, &elem_by_expand, collinfo.chain_count); chain = calloc(sizeof (collate_chain_t), collinfo.chain_count); if (chain == NULL) { fprintf(stderr, "out of memory"); return; } n = 0; RB_FOREACH(ce, elem_by_expand, &elem_by_expand) { (void) wsncpy(chain[n].str, ce->expand, COLLATE_STR_LEN); for (i = 0; i < NUM_WT; i++) { chain[n].pri[i] = get_weight(ce->ref[i], i); } n++; } if (n != collinfo.chain_count) INTERR; /* * Large (> UCHAR_MAX) character priorities */ RB_NUMNODES(collchar_t, collchars, &collchars, n); large = calloc(n, sizeof (collate_large_t)); if (large == NULL) { fprintf(stderr, "out of memory"); return; } i = 0; RB_FOREACH(cc, collchars, &collchars) { int undef = 0; /* we already gathered those */ if (cc->wc <= UCHAR_MAX) continue; for (j = 0; j < NUM_WT; j++) { if ((pri = get_weight(cc->ref[j], j)) < 0) { undef = 1; } if (undef && (pri >= 0)) { /* if undefined, then all priorities are */ INTERR; } else { large[i].pri.pri[j] = pri; } } if (!undef) { large[i].val = cc->wc; collinfo.large_count = i++; } } if ((f = open_category()) == NULL) { return; } /* Time to write the entire data set out */ if ((wr_category(vers, COLLATE_STR_LEN, f) < 0) || (wr_category(&collinfo, sizeof (collinfo), f) < 0) || (wr_category(&chars, sizeof (chars), f) < 0)) { return; } for (i = 0; i < NUM_WT; i++) { sz = sizeof (collate_subst_t) * collinfo.subst_count[i]; if (wr_category(subst[i], sz, f) < 0) { return; } } sz = sizeof (collate_chain_t) * collinfo.chain_count; if (wr_category(chain, sz, f) < 0) { return; } sz = sizeof (collate_large_t) * collinfo.large_count; if (wr_category(large, sz, f) < 0) { return; } close_category(f); } Index: head/usr.bin/localedef/wide.c =================================================================== --- head/usr.bin/localedef/wide.c (revision 290558) +++ head/usr.bin/localedef/wide.c (revision 290559) @@ -1,671 +1,666 @@ /* * Copyright 2011 Nexenta Systems, Inc. All rights reserved. * Copyright 2012 Garrett D'Amore All rights reserved. * Copyright 2015 John Marino * * This source code is derived from the illumos localedef command, and * provided under BSD-style license terms by Nexenta Systems, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * The functions in this file convert from the standard multibyte forms * to the wide character forms used internally by libc. Unfortunately, * this approach means that we need a method for each and every encoding. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include "localedef.h" static int towide_none(wchar_t *, const char *, unsigned); static int towide_utf8(wchar_t *, const char *, unsigned); static int towide_big5(wchar_t *, const char *, unsigned); static int towide_gbk(wchar_t *, const char *, unsigned); static int towide_gb2312(wchar_t *, const char *, unsigned); static int towide_gb18030(wchar_t *, const char *, unsigned); static int towide_mskanji(wchar_t *, const char *, unsigned); static int towide_euccn(wchar_t *, const char *, unsigned); static int towide_eucjp(wchar_t *, const char *, unsigned); static int towide_euckr(wchar_t *, const char *, unsigned); static int towide_euctw(wchar_t *, const char *, unsigned); static int tomb_none(char *, wchar_t); static int tomb_utf8(char *, wchar_t); static int tomb_mbs(char *, wchar_t); static int (*_towide)(wchar_t *, const char *, unsigned) = towide_none; static int (*_tomb)(char *, wchar_t) = tomb_none; static char _encoding_buffer[20] = {'N','O','N','E'}; static const char *_encoding = _encoding_buffer; static int _nbits = 7; /* * Table of supported encodings. We only bother to list the multibyte * encodings here, because single byte locales are handed by "NONE". */ static struct { const char *name; /* the name that the underlying libc implemenation uses */ const char *cname; /* the maximum number of bits required for priorities */ int nbits; int (*towide)(wchar_t *, const char *, unsigned); int (*tomb)(char *, wchar_t); } mb_encodings[] = { /* * UTF8 values max out at 0x1fffff (although in theory there could * be later extensions, but it won't happen.) This means we only need * 21 bits to be able to encode the entire range of priorities. */ { "UTF-8", "UTF-8", 21, towide_utf8, tomb_utf8 }, { "UTF8", "UTF-8", 21, towide_utf8, tomb_utf8 }, { "utf8", "UTF-8", 21, towide_utf8, tomb_utf8 }, { "utf-8", "UTF-8", 21, towide_utf8, tomb_utf8 }, { "EUC-CN", "EUC-CN", 16, towide_euccn, tomb_mbs }, { "eucCN", "EUC-CN", 16, towide_euccn, tomb_mbs }, /* * Becuase the 3-byte form of EUC-JP use the same leading byte, * only 17 bits required to provide unique priorities. (The low * bit of that first byte is set.) By setting this value low, * we can get by with only 3 bytes in the strxfrm expansion. */ { "EUC-JP", "EUC-JP", 17, towide_eucjp, tomb_mbs }, { "eucJP", "EUC-JP", 17, towide_eucjp, tomb_mbs }, { "EUC-KR", "EUC-KR", 16, towide_euckr, tomb_mbs }, { "eucKR", "EUC-KR", 16, towide_euckr, tomb_mbs }, /* * EUC-TW uses 2 bytes most of the time, but 4 bytes if the * high order byte is 0x8E. However, with 4 byte encodings, * the third byte will be A0-B0. So we only need to consider * the lower order 24 bits for collation. */ { "EUC-TW", "EUC-TW", 24, towide_euctw, tomb_mbs }, { "eucTW", "EUC-TW", 24, towide_euctw, tomb_mbs }, { "MS_Kanji", "MSKanji", 16, towide_mskanji, tomb_mbs }, { "MSKanji", "MSKanji", 16, towide_mskanji, tomb_mbs }, { "PCK", "MSKanji", 16, towide_mskanji, tomb_mbs }, { "SJIS", "MSKanji", 16, towide_mskanji, tomb_mbs }, { "Shift_JIS", "MSKanji", 16, towide_mskanji, tomb_mbs }, { "BIG5", "BIG5", 16, towide_big5, tomb_mbs }, { "big5", "BIG5", 16, towide_big5, tomb_mbs }, { "Big5", "BIG5", 16, towide_big5, tomb_mbs }, { "GBK", "GBK", 16, towide_gbk, tomb_mbs }, /* * GB18030 can get away with just 31 bits. This is because the * high order bit is always set for 4 byte values, and the * at least one of the other bits in that 4 byte value will * be non-zero. */ { "GB18030", "GB18030", 31, towide_gb18030, tomb_mbs }, /* * This should probably be an aliase for euc-cn, or vice versa. */ { "GB2312", "GB2312", 16, towide_gb2312, tomb_mbs }, { NULL, NULL, 0, 0, 0 }, }; static char * show_mb(const char *mb) { static char buf[64]; /* ASCII stuff we just print */ if (isascii(*mb) && isgraph(*mb)) { buf[0] = *mb; buf[1] = 0; return (buf); } buf[0] = 0; while (*mb != 0) { char scr[8]; (void) snprintf(scr, sizeof (scr), "\\x%02x", *mb); (void) strlcat(buf, scr, sizeof (buf)); mb++; } return (buf); } static char *widemsg; void werr(const char *fmt, ...) { char *msg; va_list va; va_start(va, fmt); (void) vasprintf(&msg, fmt, va); va_end(va); free(widemsg); widemsg = msg; } /* * This is used for 8-bit encodings. */ int towide_none(wchar_t *c, const char *mb, unsigned n __unused) { if (mb_cur_max != 1) { werr("invalid or unsupported multibyte locale"); return (-1); } *c = (uint8_t)*mb; return (1); } int tomb_none(char *mb, wchar_t wc) { if (mb_cur_max != 1) { werr("invalid or unsupported multibyte locale"); return (-1); } *(uint8_t *)mb = (wc & 0xff); mb[1] = 0; return (1); } /* * UTF-8 stores wide characters in UTF-32 form. */ int towide_utf8(wchar_t *wc, const char *mb, unsigned n) { wchar_t c; int nb; wchar_t lv; /* lowest legal value */ int i; const uint8_t *s = (const uint8_t *)mb; c = *s; if ((c & 0x80) == 0) { /* 7-bit ASCII */ *wc = c; return (1); } else if ((c & 0xe0) == 0xc0) { /* u80-u7ff - two bytes encoded */ nb = 2; lv = 0x80; c &= ~0xe0; } else if ((c & 0xf0) == 0xe0) { /* u800-uffff - three bytes encoded */ nb = 3; lv = 0x800; c &= ~0xf0; } else if ((c & 0xf8) == 0xf0) { /* u1000-u1fffff - four bytes encoded */ nb = 4; lv = 0x1000; c &= ~0xf8; } else { /* 5 and 6 byte encodings are not legal unicode */ werr("utf8 encoding too large (%s)", show_mb(mb)); return (-1); } if (nb > (int)n) { werr("incomplete utf8 sequence (%s)", show_mb(mb)); return (-1); } for (i = 1; i < nb; i++) { if (((s[i]) & 0xc0) != 0x80) { werr("illegal utf8 byte (%x)", s[i]); return (-1); } c <<= 6; c |= (s[i] & 0x3f); } if (c < lv) { werr("illegal redundant utf8 encoding (%s)", show_mb(mb)); return (-1); } *wc = c; return (nb); } int tomb_utf8(char *mb, wchar_t wc) { uint8_t *s = (uint8_t *)mb; uint8_t msk; int cnt; int i; if (wc <= 0x7f) { s[0] = wc & 0x7f; s[1] = 0; return (1); } if (wc <= 0x7ff) { cnt = 2; msk = 0xc0; } else if (wc <= 0xffff) { cnt = 3; msk = 0xe0; } else if (wc <= 0x1fffff) { cnt = 4; msk = 0xf0; } else { werr("illegal uf8 char (%x)", wc); return (-1); } for (i = cnt - 1; i; i--) { s[i] = (wc & 0x3f) | 0x80; wc >>= 6; } s[0] = (msk) | wc; s[cnt] = 0; return (cnt); } /* * Several encodings share a simplistic dual byte encoding. In these * forms, they all indicate that a two byte sequence is to be used if * the first byte has its high bit set. They all store this simple * encoding as a 16-bit value, although a great many of the possible * code points are not used in most character sets. This gives a possible * set of just over 32,000 valid code points. * * 0x00 - 0x7f - 1 byte encoding * 0x80 - 0x7fff - illegal * 0x8000 - 0xffff - 2 byte encoding */ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wcast-qual" - static int towide_dbcs(wchar_t *wc, const char *mb, unsigned n) { wchar_t c; - c = *(uint8_t *)mb; + c = *(const uint8_t *)mb; if ((c & 0x80) == 0) { /* 7-bit */ *wc = c; return (1); } if (n < 2) { werr("incomplete character sequence (%s)", show_mb(mb)); return (-1); } /* Store both bytes as a single 16-bit wide. */ c <<= 8; c |= (uint8_t)(mb[1]); *wc = c; return (2); } /* * Most multibyte locales just convert the wide character to the multibyte * form by stripping leading null bytes, and writing the 32-bit quantity * in big-endian order. */ int tomb_mbs(char *mb, wchar_t wc) { uint8_t *s = (uint8_t *)mb; int n = 0, c; if ((wc & 0xff000000U) != 0) { n = 4; } else if ((wc & 0x00ff0000U) != 0) { n = 3; } else if ((wc & 0x0000ff00U) != 0) { n = 2; } else { n = 1; } c = n; while (n) { n--; s[n] = wc & 0xff; wc >>= 8; } /* ensure null termination */ s[c] = 0; return (c); } /* * big5 is a simple dual byte character set. */ int towide_big5(wchar_t *wc, const char *mb, unsigned n) { return (towide_dbcs(wc, mb, n)); } /* * GBK encodes wides in the same way that big5 does, the high order * bit of the first byte indicates a double byte character. */ int towide_gbk(wchar_t *wc, const char *mb, unsigned n) { return (towide_dbcs(wc, mb, n)); } /* * GB2312 is another DBCS. Its cleaner than others in that the second * byte does not encode ASCII, but it supports characters. */ int towide_gb2312(wchar_t *wc, const char *mb, unsigned n) { return (towide_dbcs(wc, mb, n)); } /* * GB18030. This encodes as 8, 16, or 32-bits. * 7-bit values are in 1 byte, 4 byte sequences are used when * the second byte encodes 0x30-39 and all other sequences are 2 bytes. */ int towide_gb18030(wchar_t *wc, const char *mb, unsigned n) { wchar_t c; - c = *(uint8_t *)mb; + c = *(const uint8_t *)mb; if ((c & 0x80) == 0) { /* 7-bit */ *wc = c; return (1); } if (n < 2) { werr("incomplete character sequence (%s)", show_mb(mb)); return (-1); } /* pull in the second byte */ c <<= 8; c |= (uint8_t)(mb[1]); if (((c & 0xff) >= 0x30) && ((c & 0xff) <= 0x39)) { if (n < 4) { werr("incomplete 4-byte character sequence (%s)", show_mb(mb)); return (-1); } c <<= 8; c |= (uint8_t)(mb[2]); c <<= 8; c |= (uint8_t)(mb[3]); *wc = c; return (4); } *wc = c; return (2); } /* * MS-Kanji (aka SJIS) is almost a clean DBCS like the others, but it * also has a range of single byte characters above 0x80. (0xa1-0xdf). */ int towide_mskanji(wchar_t *wc, const char *mb, unsigned n) { wchar_t c; - c = *(uint8_t *)mb; + c = *(const uint8_t *)mb; if ((c < 0x80) || ((c > 0xa0) && (c < 0xe0))) { /* 7-bit */ *wc = c; return (1); } if (n < 2) { werr("incomplete character sequence (%s)", show_mb(mb)); return (-1); } /* Store both bytes as a single 16-bit wide. */ c <<= 8; c |= (uint8_t)(mb[1]); *wc = c; return (2); } /* * EUC forms. EUC encodings are "variable". FreeBSD carries some additional * variable data to encode these, but we're going to treat each as independent * instead. Its the only way we can sensibly move forward. * * Note that the way in which the different EUC forms vary is how wide * CS2 and CS3 are and what the first byte of them is. */ static int towide_euc_impl(wchar_t *wc, const char *mb, unsigned n, uint8_t cs2, uint8_t cs2width, uint8_t cs3, uint8_t cs3width) { int i; int width = 2; wchar_t c; - c = *(uint8_t *)mb; + c = *(const uint8_t *)mb; /* * All variations of EUC encode 7-bit ASCII as one byte, and use * additional bytes for more than that. */ if ((c & 0x80) == 0) { /* 7-bit */ *wc = c; return (1); } /* * All EUC variants reserve 0xa1-0xff to identify CS1, which * is always two bytes wide. Note that unused CS will be zero, * and that cannot be true because we know that the high order * bit must be set. */ if (c >= 0xa1) { width = 2; } else if (c == cs2) { width = cs2width; } else if (c == cs3) { width = cs3width; } if ((int)n < width) { werr("incomplete character sequence (%s)", show_mb(mb)); return (-1); } for (i = 1; i < width; i++) { /* pull in the next byte */ c <<= 8; c |= (uint8_t)(mb[i]); } *wc = c; return (width); } - -#pragma GCC diagnostic pop /* * EUC-CN encodes as follows: * * Code set 0 (ASCII): 0x21-0x7E * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE * Code set 2: unused * Code set 3: unused */ int towide_euccn(wchar_t *wc, const char *mb, unsigned n) { return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0)); } /* * EUC-JP encodes as follows: * * Code set 0 (ASCII or JIS X 0201-1976 Roman): 0x21-0x7E * Code set 1 (JIS X 0208): 0xA1A1-0xFEFE * Code set 2 (half-width katakana): 0x8EA1-0x8EDF * Code set 3 (JIS X 0212-1990): 0x8FA1A1-0x8FFEFE */ int towide_eucjp(wchar_t *wc, const char *mb, unsigned n) { return (towide_euc_impl(wc, mb, n, 0x8e, 2, 0x8f, 3)); } /* * EUC-KR encodes as follows: * * Code set 0 (ASCII or KS C 5636-1993): 0x21-0x7E * Code set 1 (KS C 5601-1992): 0xA1A1-0xFEFE * Code set 2: unused * Code set 3: unused */ int towide_euckr(wchar_t *wc, const char *mb, unsigned n) { return (towide_euc_impl(wc, mb, n, 0, 0, 0, 0)); } /* * EUC-TW encodes as follows: * * Code set 0 (ASCII): 0x21-0x7E * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE * Code set 2 (CNS 11643-1992 Planes 1-16): 0x8EA1A1A1-0x8EB0FEFE * Code set 3: unused */ int towide_euctw(wchar_t *wc, const char *mb, unsigned n) { return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0)); } /* * Public entry points. */ int to_wide(wchar_t *wc, const char *mb) { /* this won't fail hard */ return (_towide(wc, mb, strlen(mb))); } int to_mb(char *mb, wchar_t wc) { int rv; if ((rv = _tomb(mb, wc)) < 0) { errf(widemsg); free(widemsg); widemsg = NULL; } return (rv); } char * to_mb_string(const wchar_t *wcs) { char *mbs; char *ptr; int len; mbs = malloc((wcslen(wcs) * mb_cur_max) + 1); if (mbs == NULL) { errf("out of memory"); return (NULL); } ptr = mbs; while (*wcs) { if ((len = to_mb(ptr, *wcs)) < 0) { INTERR; free(mbs); return (NULL); } wcs++; ptr += len; } *ptr = 0; return (mbs); } void set_wide_encoding(const char *encoding) { int i; _towide = towide_none; _tomb = tomb_none; _nbits = 8; snprintf(_encoding_buffer, sizeof(_encoding_buffer), "NONE:%s", encoding); for (i = 0; mb_encodings[i].name; i++) { if (strcasecmp(encoding, mb_encodings[i].name) == 0) { _towide = mb_encodings[i].towide; _tomb = mb_encodings[i].tomb; _encoding = mb_encodings[i].cname; _nbits = mb_encodings[i].nbits; break; } } } const char * get_wide_encoding(void) { return (_encoding); } int max_wide(void) { return ((int)((1U << _nbits) - 1)); }