Index: projects/collation/lib/libc/locale/nl_langinfo.c =================================================================== --- projects/collation/lib/libc/locale/nl_langinfo.c (revision 290232) +++ projects/collation/lib/libc/locale/nl_langinfo.c (revision 290233) @@ -1,193 +1,207 @@ /*- * Copyright (c) 2001, 2003 Alexey Zelkin * All rights reserved. * * Copyright (c) 2011 The FreeBSD Foundation * All rights reserved. * Portions of this software were developed by David Chisnall * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include +#include +#include +#include "mblocal.h" #include "lnumeric.h" #include "lmessages.h" #include "lmonetary.h" #include "../stdtime/timelocal.h" #define _REL(BASE) ((int)item-BASE) char * nl_langinfo_l(nl_item item, locale_t loc) { char *ret, *cs; const char *s; FIX_LOCALE(loc); switch (item) { case CODESET: - ret = ""; - if ((s = querylocale(LC_CTYPE_MASK, loc)) != NULL) { - if ((cs = strchr(s, '.')) != NULL) - ret = cs + 1; - else if (strcmp(s, "C") == 0 || - strcmp(s, "POSIX") == 0) - ret = "US-ASCII"; - } + s = XLOCALE_CTYPE(loc)->runes->__encoding; + if (strcmp(s, "EUC-CN") == 0) + ret = "eucCN"; + else if (strcmp(s, "EUC-JP") == 0) + ret = "eucJP"; + else if (strcmp(s, "EUC-KR") == 0) + ret = "eucKR"; + else if (strcmp(s, "EUC-TW") == 0) + ret = "eucTW"; + else if (strcmp(s, "BIG5") == 0) + ret = "Big5"; + else if (strcmp(s, "MSKanji") == 0) + ret = "SJIS"; + else if (strcmp(s, "NONE") == 0) + ret = "US-ASCII"; + else if (strncmp(s, "NONE:", 5) == 0) + ret = (char *)(s + 5); + else + ret = (char *)s; break; case D_T_FMT: ret = (char *) __get_current_time_locale(loc)->c_fmt; break; case D_FMT: ret = (char *) __get_current_time_locale(loc)->x_fmt; break; case T_FMT: ret = (char *) __get_current_time_locale(loc)->X_fmt; break; case T_FMT_AMPM: ret = (char *) __get_current_time_locale(loc)->ampm_fmt; break; case AM_STR: ret = (char *) __get_current_time_locale(loc)->am; break; case PM_STR: ret = (char *) __get_current_time_locale(loc)->pm; break; case DAY_1: case DAY_2: case DAY_3: case DAY_4: case DAY_5: case DAY_6: case DAY_7: ret = (char*) __get_current_time_locale(loc)->weekday[_REL(DAY_1)]; break; case ABDAY_1: case ABDAY_2: case ABDAY_3: case ABDAY_4: case ABDAY_5: case ABDAY_6: case ABDAY_7: ret = (char*) __get_current_time_locale(loc)->wday[_REL(ABDAY_1)]; break; case MON_1: case MON_2: case MON_3: case MON_4: case MON_5: case MON_6: case MON_7: case MON_8: case MON_9: case MON_10: case MON_11: case MON_12: ret = (char*) __get_current_time_locale(loc)->month[_REL(MON_1)]; break; case ABMON_1: case ABMON_2: case ABMON_3: case ABMON_4: case ABMON_5: case ABMON_6: case ABMON_7: case ABMON_8: case ABMON_9: case ABMON_10: case ABMON_11: case ABMON_12: ret = (char*) __get_current_time_locale(loc)->mon[_REL(ABMON_1)]; break; case ALTMON_1: case ALTMON_2: case ALTMON_3: case ALTMON_4: case ALTMON_5: case ALTMON_6: case ALTMON_7: case ALTMON_8: case ALTMON_9: case ALTMON_10: case ALTMON_11: case ALTMON_12: ret = (char*) __get_current_time_locale(loc)->alt_month[_REL(ALTMON_1)]; break; case ERA: /* XXX: need to be implemented */ ret = ""; break; case ERA_D_FMT: /* XXX: need to be implemented */ ret = ""; break; case ERA_D_T_FMT: /* XXX: need to be implemented */ ret = ""; break; case ERA_T_FMT: /* XXX: need to be implemented */ ret = ""; break; case ALT_DIGITS: /* XXX: need to be implemented */ ret = ""; break; case RADIXCHAR: ret = (char*) __get_current_numeric_locale(loc)->decimal_point; break; case THOUSEP: ret = (char*) __get_current_numeric_locale(loc)->thousands_sep; break; case YESEXPR: ret = (char*) __get_current_messages_locale(loc)->yesexpr; break; case NOEXPR: ret = (char*) __get_current_messages_locale(loc)->noexpr; break; /* * YESSTR and NOSTR items marked with LEGACY are available, but not * recomended by SUSv2 to be used in portable applications since * they're subject to remove in future specification editions. */ case YESSTR: /* LEGACY */ ret = (char*) __get_current_messages_locale(loc)->yesstr; break; case NOSTR: /* LEGACY */ ret = (char*) __get_current_messages_locale(loc)->nostr; break; /* * SUSv2 special formatted currency string */ case CRNCYSTR: ret = ""; cs = (char*) __get_current_monetary_locale(loc)->currency_symbol; if (*cs != '\0') { char pos = localeconv_l(loc)->p_cs_precedes; if (pos == localeconv_l(loc)->n_cs_precedes) { char psn = '\0'; if (pos == CHAR_MAX) { if (strcmp(cs, __get_current_monetary_locale(loc)->mon_decimal_point) == 0) psn = '.'; } else psn = pos ? '-' : '+'; if (psn != '\0') { int clen = strlen(cs); if ((loc->csym = reallocf(loc->csym, clen + 2)) != NULL) { *loc->csym = psn; strcpy(loc->csym + 1, cs); ret = loc->csym; } } } } break; case D_MD_ORDER: /* FreeBSD local extension */ ret = (char *) __get_current_time_locale(loc)->md_order; break; default: ret = ""; } return (ret); } char * nl_langinfo(nl_item item) { return nl_langinfo_l(item, __get_locale()); } Index: projects/collation/lib/libc/locale/setrunelocale.c =================================================================== --- projects/collation/lib/libc/locale/setrunelocale.c (revision 290232) +++ projects/collation/lib/libc/locale/setrunelocale.c (revision 290233) @@ -1,211 +1,211 @@ /*- * Copyright (c) 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * Paul Borman at Krystal Technologies. * * Copyright (c) 2011 The FreeBSD Foundation * All rights reserved. * Portions of this software were developed by David Chisnall * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #define __RUNETYPE_INTERNAL 1 #include #include #include #include #include #include #include #include #include "ldpart.h" #include "mblocal.h" #include "setlocale.h" #undef _CurrentRuneLocale extern _RuneLocale const *_CurrentRuneLocale; #ifndef __NO_TLS /* * A cached version of the runes for this thread. Used by ctype.h */ _Thread_local const _RuneLocale *_ThreadRuneLocale; #endif extern int __mb_sb_limit; extern _RuneLocale *_Read_RuneMagi(const char *); static int __setrunelocale(struct xlocale_ctype *l, const char *); static void destruct_ctype(void *v) { struct xlocale_ctype *l = v; if (&_DefaultRuneLocale != l->runes) free(l->runes); free(l); } const _RuneLocale * __getCurrentRuneLocale(void) { return XLOCALE_CTYPE(__get_locale())->runes; } static void free_runes(_RuneLocale *rl) { if ((rl != &_DefaultRuneLocale) && (rl)) { free(rl); } } static int __setrunelocale(struct xlocale_ctype *l, const char *encoding) { _RuneLocale *rl; int ret; char *path; struct xlocale_ctype saved = *l; /* * The "C" and "POSIX" locale are always here. */ if (strcmp(encoding, "C") == 0 || strcmp(encoding, "POSIX") == 0) { free_runes(saved.runes); (void) _none_init(l, (_RuneLocale*)&_DefaultRuneLocale); return (0); } /* Range checking not needed, encoding length already checked before */ asprintf(&path, "%s/%s/LC_CTYPE", _PathLocale, encoding); if (path == NULL) return (0); if ((rl = _Read_RuneMagi(path)) == NULL) { free(path); errno = EINVAL; return (errno); } free(path); l->__mbrtowc = NULL; l->__mbsinit = NULL; l->__mbsnrtowcs = NULL; l->__wcrtomb = NULL; l->__wcsnrtombs = NULL; rl->__sputrune = NULL; rl->__sgetrune = NULL; - if (strcmp(rl->__encoding, "NONE") == 0) + if (strncmp(rl->__encoding, "NONE", 4) == 0) ret = _none_init(l, rl); else if (strcmp(rl->__encoding, "UTF-8") == 0) ret = _UTF8_init(l, rl); else if (strcmp(rl->__encoding, "EUC-CN") == 0) ret = _EUC_CN_init(l, rl); else if (strcmp(rl->__encoding, "EUC-JP") == 0) ret = _EUC_JP_init(l, rl); else if (strcmp(rl->__encoding, "EUC-KR") == 0) ret = _EUC_KR_init(l, rl); else if (strcmp(rl->__encoding, "EUC-TW") == 0) ret = _EUC_TW_init(l, rl); else if (strcmp(rl->__encoding, "GB18030") == 0) ret = _GB18030_init(l, rl); else if (strcmp(rl->__encoding, "GB2312") == 0) ret = _GB2312_init(l, rl); else if (strcmp(rl->__encoding, "GBK") == 0) ret = _GBK_init(l, rl); else if (strcmp(rl->__encoding, "BIG5") == 0) ret = _BIG5_init(l, rl); else if (strcmp(rl->__encoding, "MSKanji") == 0) ret = _MSKanji_init(l, rl); else ret = EFTYPE; if (ret == 0) { /* Free the old runes if it exists. */ free_runes(saved.runes); } else { /* Restore the saved version if this failed. */ memcpy(l, &saved, sizeof(struct xlocale_ctype)); free(rl); } return (ret); } int __wrap_setrunelocale(const char *locale) { int ret = __setrunelocale(&__xlocale_global_ctype, locale); if (ret != 0) { errno = ret; return (_LDP_ERROR); } __mb_cur_max = __xlocale_global_ctype.__mb_cur_max; __mb_sb_limit = __xlocale_global_ctype.__mb_sb_limit; _CurrentRuneLocale = __xlocale_global_ctype.runes; return (_LDP_LOADED); } #ifndef __NO_TLS void __set_thread_rune_locale(locale_t loc) { if (loc == NULL) { _ThreadRuneLocale = &_DefaultRuneLocale; } else if (loc == LC_GLOBAL_LOCALE) { _ThreadRuneLocale = 0; } else { _ThreadRuneLocale = XLOCALE_CTYPE(loc)->runes; } } #endif void * __ctype_load(const char *locale, locale_t unused __unused) { struct xlocale_ctype *l = calloc(sizeof(struct xlocale_ctype), 1); l->header.header.destructor = destruct_ctype; if (__setrunelocale(l, locale)) { free(l); return NULL; } return l; } Index: projects/collation/usr.bin/localedef/wide.c =================================================================== --- projects/collation/usr.bin/localedef/wide.c (revision 290232) +++ projects/collation/usr.bin/localedef/wide.c (revision 290233) @@ -1,669 +1,669 @@ /* * Copyright 2011 Nexenta Systems, Inc. All rights reserved. * Copyright 2012 Garrett D'Amore All rights reserved. * Copyright 2015 John Marino * * This source code is derived from the illumos localedef command, and * provided under BSD-style license terms by Nexenta Systems, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * The functions in this file convert from the standard multibyte forms * to the wide character forms used internally by libc. Unfortunately, * this approach means that we need a method for each and every encoding. */ #include __FBSDID("$FreeBSD$"); -#include #include #include #include #include #include "localedef.h" static int towide_none(wchar_t *, const char *, unsigned); static int towide_utf8(wchar_t *, const char *, unsigned); static int towide_big5(wchar_t *, const char *, unsigned); static int towide_gbk(wchar_t *, const char *, unsigned); static int towide_gb2312(wchar_t *, const char *, unsigned); static int towide_gb18030(wchar_t *, const char *, unsigned); static int towide_mskanji(wchar_t *, const char *, unsigned); static int towide_euccn(wchar_t *, const char *, unsigned); static int towide_eucjp(wchar_t *, const char *, unsigned); static int towide_euckr(wchar_t *, const char *, unsigned); static int towide_euctw(wchar_t *, const char *, unsigned); static int tomb_none(char *, wchar_t); static int tomb_utf8(char *, wchar_t); static int tomb_mbs(char *, wchar_t); static int (*_towide)(wchar_t *, const char *, unsigned) = towide_none; static int (*_tomb)(char *, wchar_t) = tomb_none; -static const char *_encoding = "NONE"; +static char _encoding_buffer[20] = {'N','O','N','E'}; +static const char *_encoding = _encoding_buffer; static int _nbits = 7; /* * Table of supported encodings. We only bother to list the multibyte * encodings here, because single byte locales are handed by "NONE". */ static struct { const char *name; /* the name that the underlying libc implemenation uses */ const char *cname; /* the maximum number of bits required for priorities */ int nbits; int (*towide)(wchar_t *, const char *, unsigned); int (*tomb)(char *, wchar_t); } mb_encodings[] = { /* * UTF8 values max out at 0x1fffff (although in theory there could * be later extensions, but it won't happen.) This means we only need * 21 bits to be able to encode the entire range of priorities. */ { "UTF-8", "UTF-8", 21, towide_utf8, tomb_utf8 }, { "UTF8", "UTF-8", 21, towide_utf8, tomb_utf8 }, { "utf8", "UTF-8", 21, towide_utf8, tomb_utf8 }, { "utf-8", "UTF-8", 21, towide_utf8, tomb_utf8 }, { "EUC-CN", "EUC-CN", 16, towide_euccn, tomb_mbs }, { "eucCN", "EUC-CN", 16, towide_euccn, tomb_mbs }, /* * Becuase the 3-byte form of EUC-JP use the same leading byte, * only 17 bits required to provide unique priorities. (The low * bit of that first byte is set.) By setting this value low, * we can get by with only 3 bytes in the strxfrm expansion. */ { "EUC-JP", "EUC-JP", 17, towide_eucjp, tomb_mbs }, { "eucJP", "EUC-JP", 17, towide_eucjp, tomb_mbs }, { "EUC-KR", "EUC-KR", 16, towide_euckr, tomb_mbs }, { "eucKR", "EUC-KR", 16, towide_euckr, tomb_mbs }, /* * EUC-TW uses 2 bytes most of the time, but 4 bytes if the * high order byte is 0x8E. However, with 4 byte encodings, * the third byte will be A0-B0. So we only need to consider * the lower order 24 bits for collation. */ { "EUC-TW", "EUC-TW", 24, towide_euctw, tomb_mbs }, { "eucTW", "EUC-TW", 24, towide_euctw, tomb_mbs }, { "MS_Kanji", "MSKanji", 16, towide_mskanji, tomb_mbs }, { "MSKanji", "MSKanji", 16, towide_mskanji, tomb_mbs }, { "PCK", "MSKanji", 16, towide_mskanji, tomb_mbs }, { "SJIS", "MSKanji", 16, towide_mskanji, tomb_mbs }, { "Shift_JIS", "MSKanji", 16, towide_mskanji, tomb_mbs }, { "BIG5", "BIG5", 16, towide_big5, tomb_mbs }, { "big5", "BIG5", 16, towide_big5, tomb_mbs }, { "Big5", "BIG5", 16, towide_big5, tomb_mbs }, { "GBK", "GBK", 16, towide_gbk, tomb_mbs }, /* * GB18030 can get away with just 31 bits. This is because the * high order bit is always set for 4 byte values, and the * at least one of the other bits in that 4 byte value will * be non-zero. */ { "GB18030", "GB18030", 31, towide_gb18030, tomb_mbs }, /* * This should probably be an aliase for euc-cn, or vice versa. */ { "GB2312", "GB2312", 16, towide_gb2312, tomb_mbs }, { NULL, NULL, 0, 0, 0 }, }; static char * show_mb(const char *mb) { static char buf[64]; /* ASCII stuff we just print */ if (isascii(*mb) && isgraph(*mb)) { buf[0] = *mb; buf[1] = 0; return (buf); } buf[0] = 0; while (*mb != 0) { char scr[8]; (void) snprintf(scr, sizeof (scr), "\\x%02x", *mb); (void) strlcat(buf, scr, sizeof (buf)); mb++; } return (buf); } static char *widemsg; void werr(const char *fmt, ...) { char *msg; va_list va; va_start(va, fmt); (void) vasprintf(&msg, fmt, va); va_end(va); free(widemsg); widemsg = msg; } /* * This is used for 8-bit encodings. */ int towide_none(wchar_t *c, const char *mb, unsigned n __unused) { if (mb_cur_max != 1) { werr("invalid or unsupported multibyte locale"); return (-1); } *c = (uint8_t)*mb; return (1); } int tomb_none(char *mb, wchar_t wc) { if (mb_cur_max != 1) { werr("invalid or unsupported multibyte locale"); return (-1); } *(uint8_t *)mb = (wc & 0xff); mb[1] = 0; return (1); } /* * UTF-8 stores wide characters in UTF-32 form. */ int towide_utf8(wchar_t *wc, const char *mb, unsigned n) { wchar_t c; int nb; int lv; /* lowest legal value */ int i; const uint8_t *s = (const uint8_t *)mb; c = *s; if ((c & 0x80) == 0) { /* 7-bit ASCII */ *wc = c; return (1); } else if ((c & 0xe0) == 0xc0) { /* u80-u7ff - two bytes encoded */ nb = 2; lv = 0x80; c &= ~0xe0; } else if ((c & 0xf0) == 0xe0) { /* u800-uffff - three bytes encoded */ nb = 3; lv = 0x800; c &= ~0xf0; } else if ((c & 0xf8) == 0xf0) { /* u1000-u1fffff - four bytes encoded */ nb = 4; lv = 0x1000; c &= ~0xf8; } else { /* 5 and 6 byte encodings are not legal unicode */ werr("utf8 encoding too large (%s)", show_mb(mb)); return (-1); } if (nb > (int)n) { werr("incomplete utf8 sequence (%s)", show_mb(mb)); return (-1); } for (i = 1; i < nb; i++) { if (((s[i]) & 0xc0) != 0x80) { werr("illegal utf8 byte (%x)", s[i]); return (-1); } c <<= 6; c |= (s[i] & 0x3f); } if (c < lv) { werr("illegal redundant utf8 encoding (%s)", show_mb(mb)); return (-1); } *wc = c; return (nb); } int tomb_utf8(char *mb, wchar_t wc) { uint8_t *s = (uint8_t *)mb; uint8_t msk; int cnt; int i; if (wc <= 0x7f) { s[0] = wc & 0x7f; s[1] = 0; return (1); } if (wc <= 0x7ff) { cnt = 2; msk = 0xc0; } else if (wc <= 0xffff) { cnt = 3; msk = 0xe0; } else if (wc <= 0x1fffff) { cnt = 4; msk = 0xf0; } else { werr("illegal uf8 char (%x)", wc); return (-1); } for (i = cnt - 1; i; i--) { s[i] = (wc & 0x3f) | 0x80; wc >>= 6; } s[0] = (msk) | wc; s[cnt] = 0; return (cnt); } /* * Several encodings share a simplistic dual byte encoding. In these * forms, they all indicate that a two byte sequence is to be used if * the first byte has its high bit set. They all store this simple * encoding as a 16-bit value, although a great many of the possible * code points are not used in most character sets. This gives a possible * set of just over 32,000 valid code points. * * 0x00 - 0x7f - 1 byte encoding * 0x80 - 0x7fff - illegal * 0x8000 - 0xffff - 2 byte encoding */ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-qual" static int towide_dbcs(wchar_t *wc, const char *mb, unsigned n) { wchar_t c; c = *(uint8_t *)mb; if ((c & 0x80) == 0) { /* 7-bit */ *wc = c; return (1); } if (n < 2) { werr("incomplete character sequence (%s)", show_mb(mb)); return (-1); } /* Store both bytes as a single 16-bit wide. */ c <<= 8; c |= (uint8_t)(mb[1]); *wc = c; return (2); } /* * Most multibyte locales just convert the wide character to the multibyte * form by stripping leading null bytes, and writing the 32-bit quantity * in big-endian order. */ int tomb_mbs(char *mb, wchar_t wc) { uint8_t *s = (uint8_t *)mb; int n = 0, c; if ((wc & 0xff000000U) != 0) { n = 4; } else if ((wc & 0x00ff0000U) != 0) { n = 3; } else if ((wc & 0x0000ff00U) != 0) { n = 2; } else { n = 1; } c = n; while (n) { n--; s[n] = wc & 0xff; wc >>= 8; } /* ensure null termination */ s[c] = 0; return (c); } /* * big5 is a simple dual byte character set. */ int towide_big5(wchar_t *wc, const char *mb, unsigned n) { return (towide_dbcs(wc, mb, n)); } /* * GBK encodes wides in the same way that big5 does, the high order * bit of the first byte indicates a double byte character. */ int towide_gbk(wchar_t *wc, const char *mb, unsigned n) { return (towide_dbcs(wc, mb, n)); } /* * GB2312 is another DBCS. Its cleaner than others in that the second * byte does not encode ASCII, but it supports characters. */ int towide_gb2312(wchar_t *wc, const char *mb, unsigned n) { return (towide_dbcs(wc, mb, n)); } /* * GB18030. This encodes as 8, 16, or 32-bits. * 7-bit values are in 1 byte, 4 byte sequences are used when * the second byte encodes 0x30-39 and all other sequences are 2 bytes. */ int towide_gb18030(wchar_t *wc, const char *mb, unsigned n) { wchar_t c; c = *(uint8_t *)mb; if ((c & 0x80) == 0) { /* 7-bit */ *wc = c; return (1); } if (n < 2) { werr("incomplete character sequence (%s)", show_mb(mb)); return (-1); } /* pull in the second byte */ c <<= 8; c |= (uint8_t)(mb[1]); if (((c & 0xff) >= 0x30) && ((c & 0xff) <= 0x39)) { if (n < 4) { werr("incomplete 4-byte character sequence (%s)", show_mb(mb)); return (-1); } c <<= 8; c |= (uint8_t)(mb[2]); c <<= 8; c |= (uint8_t)(mb[3]); *wc = c; return (4); } *wc = c; return (2); } /* * MS-Kanji (aka SJIS) is almost a clean DBCS like the others, but it * also has a range of single byte characters above 0x80. (0xa1-0xdf). */ int towide_mskanji(wchar_t *wc, const char *mb, unsigned n) { wchar_t c; c = *(uint8_t *)mb; if ((c < 0x80) || ((c > 0xa0) && (c < 0xe0))) { /* 7-bit */ *wc = c; return (1); } if (n < 2) { werr("incomplete character sequence (%s)", show_mb(mb)); return (-1); } /* Store both bytes as a single 16-bit wide. */ c <<= 8; c |= (uint8_t)(mb[1]); *wc = c; return (2); } /* * EUC forms. EUC encodings are "variable". FreeBSD carries some additional * variable data to encode these, but we're going to treat each as independent * instead. Its the only way we can sensibly move forward. * * Note that the way in which the different EUC forms vary is how wide * CS2 and CS3 are and what the first byte of them is. */ static int towide_euc_impl(wchar_t *wc, const char *mb, unsigned n, uint8_t cs2, uint8_t cs2width, uint8_t cs3, uint8_t cs3width) { int i; int width = 2; wchar_t c; c = *(uint8_t *)mb; /* * All variations of EUC encode 7-bit ASCII as one byte, and use * additional bytes for more than that. */ if ((c & 0x80) == 0) { /* 7-bit */ *wc = c; return (1); } /* * All EUC variants reserve 0xa1-0xff to identify CS1, which * is always two bytes wide. Note that unused CS will be zero, * and that cannot be true because we know that the high order * bit must be set. */ if (c >= 0xa1) { width = 2; } else if (c == cs2) { width = cs2width; } else if (c == cs3) { width = cs3width; } if ((int)n < width) { werr("incomplete character sequence (%s)", show_mb(mb)); return (-1); } for (i = 1; i < width; i++) { /* pull in the next byte */ c <<= 8; c |= (uint8_t)(mb[i]); } *wc = c; return (width); } #pragma GCC diagnostic pop /* * EUC-CN encodes as follows: * * Code set 0 (ASCII): 0x21-0x7E * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE * Code set 2: unused * Code set 3: unused */ int towide_euccn(wchar_t *wc, const char *mb, unsigned n) { return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0)); } /* * EUC-JP encodes as follows: * * Code set 0 (ASCII or JIS X 0201-1976 Roman): 0x21-0x7E * Code set 1 (JIS X 0208): 0xA1A1-0xFEFE * Code set 2 (half-width katakana): 0x8EA1-0x8EDF * Code set 3 (JIS X 0212-1990): 0x8FA1A1-0x8FFEFE */ int towide_eucjp(wchar_t *wc, const char *mb, unsigned n) { return (towide_euc_impl(wc, mb, n, 0x8e, 2, 0x8f, 3)); } /* * EUC-KR encodes as follows: * * Code set 0 (ASCII or KS C 5636-1993): 0x21-0x7E * Code set 1 (KS C 5601-1992): 0xA1A1-0xFEFE * Code set 2: unused * Code set 3: unused */ int towide_euckr(wchar_t *wc, const char *mb, unsigned n) { return (towide_euc_impl(wc, mb, n, 0, 0, 0, 0)); } /* * EUC-TW encodes as follows: * * Code set 0 (ASCII): 0x21-0x7E * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE * Code set 2 (CNS 11643-1992 Planes 1-16): 0x8EA1A1A1-0x8EB0FEFE * Code set 3: unused */ int towide_euctw(wchar_t *wc, const char *mb, unsigned n) { return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0)); } /* * Public entry points. */ int to_wide(wchar_t *wc, const char *mb) { /* this won't fail hard */ return (_towide(wc, mb, strlen(mb))); } int to_mb(char *mb, wchar_t wc) { int rv; if ((rv = _tomb(mb, wc)) < 0) { errf(widemsg); free(widemsg); widemsg = NULL; } return (rv); } char * to_mb_string(const wchar_t *wcs) { char *mbs; char *ptr; int len; mbs = malloc((wcslen(wcs) * mb_cur_max) + 1); if (mbs == NULL) { errf("out of memory"); return (NULL); } ptr = mbs; while (*wcs) { if ((len = to_mb(ptr, *wcs)) < 0) { INTERR; free(mbs); return (NULL); } wcs++; ptr += len; } *ptr = 0; return (mbs); } void set_wide_encoding(const char *encoding) { int i; _towide = towide_none; _tomb = tomb_none; - _encoding = "NONE"; _nbits = 8; + snprint(_encoding_buffer, sizeof(_encoding_buffer), "NONE:%s", encoding); for (i = 0; mb_encodings[i].name; i++) { if (strcasecmp(encoding, mb_encodings[i].name) == 0) { _towide = mb_encodings[i].towide; _tomb = mb_encodings[i].tomb; _encoding = mb_encodings[i].cname; _nbits = mb_encodings[i].nbits; break; } } } const char * get_wide_encoding(void) { return (_encoding); } int max_wide(void) { return ((int)((1U << _nbits) - 1)); }