Index: head/usr.bin/localedef/localedef.c =================================================================== --- head/usr.bin/localedef/localedef.c (revision 318277) +++ head/usr.bin/localedef/localedef.c (revision 318278) @@ -1,346 +1,346 @@ /*- * Copyright 2010 Nexenta Systems, Inc. All rights reserved. * Copyright 2015 John Marino * * This source code is derived from the illumos localedef command, and * provided under BSD-style license terms by Nexenta Systems, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * POSIX localedef. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include "localedef.h" #include "parser.h" #ifndef TEXT_DOMAIN #define TEXT_DOMAIN "SYS_TEST" #endif static int bsd = 0; int verbose = 0; int undefok = 0; int warnok = 0; static char *locname = NULL; static char locpath[PATH_MAX]; const char * category_name(void) { switch (get_category()) { case T_CHARMAP: return ("CHARMAP"); case T_WIDTH: return ("WIDTH"); case T_COLLATE: return ("LC_COLLATE"); case T_CTYPE: return ("LC_CTYPE"); case T_MESSAGES: return ("LC_MESSAGES"); case T_MONETARY: return ("LC_MONETARY"); case T_NUMERIC: return ("LC_NUMERIC"); case T_TIME: return ("LC_TIME"); default: INTERR; return (NULL); } } static char * category_file(void) { if (bsd) (void) snprintf(locpath, sizeof (locpath), "%s.%s", locname, category_name()); else (void) snprintf(locpath, sizeof (locpath), "%s/%s", locname, category_name()); return (locpath); } FILE * open_category(void) { FILE *file; if (verbose) { (void) printf("Writing category %s: ", category_name()); (void) fflush(stdout); } /* make the parent directory */ if (!bsd) (void) mkdir(dirname(category_file()), 0755); /* * note that we have to regenerate the file name, as dirname * clobbered it. */ file = fopen(category_file(), "w"); if (file == NULL) { - errf(strerror(errno)); + errf("%s", strerror(errno)); return (NULL); } return (file); } void close_category(FILE *f) { if (fchmod(fileno(f), 0644) < 0) { (void) fclose(f); (void) unlink(category_file()); - errf(strerror(errno)); + errf("%s", strerror(errno)); } if (fclose(f) < 0) { (void) unlink(category_file()); - errf(strerror(errno)); + errf("%s", strerror(errno)); } if (verbose) { (void) fprintf(stdout, "done.\n"); (void) fflush(stdout); } } /* * This function is used when copying the category from another * locale. Note that the copy is actually performed using a hard * link for efficiency. */ void copy_category(char *src) { char srcpath[PATH_MAX]; int rv; (void) snprintf(srcpath, sizeof (srcpath), "%s/%s", src, category_name()); rv = access(srcpath, R_OK); if ((rv != 0) && (strchr(srcpath, '/') == NULL)) { /* Maybe we should try the system locale */ (void) snprintf(srcpath, sizeof (srcpath), "/usr/lib/locale/%s/%s", src, category_name()); rv = access(srcpath, R_OK); } if (rv != 0) { fprintf(stderr,"source locale data unavailable: %s", src); return; } if (verbose > 1) { (void) printf("Copying category %s from %s: ", category_name(), src); (void) fflush(stdout); } /* make the parent directory */ if (!bsd) (void) mkdir(dirname(category_file()), 0755); if (link(srcpath, category_file()) != 0) { fprintf(stderr,"unable to copy locale data: %s", strerror(errno)); return; } if (verbose > 1) { (void) printf("done.\n"); } } int putl_category(const char *s, FILE *f) { if (s && fputs(s, f) == EOF) { (void) fclose(f); (void) unlink(category_file()); - errf(strerror(errno)); + errf("%s", strerror(errno)); return (EOF); } if (fputc('\n', f) == EOF) { (void) fclose(f); (void) unlink(category_file()); - errf(strerror(errno)); + errf("%s", strerror(errno)); return (EOF); } return (0); } int wr_category(void *buf, size_t sz, FILE *f) { if (!sz) { return (0); } if (fwrite(buf, sz, 1, f) < 1) { (void) fclose(f); (void) unlink(category_file()); - errf(strerror(errno)); + errf("%s", strerror(errno)); return (EOF); } return (0); } int yyparse(void); static void usage(void) { (void) fprintf(stderr, "Usage: localedef [options] localename\n"); (void) fprintf(stderr, "[options] are:\n"); (void) fprintf(stderr, " -D : BSD-style output\n"); (void) fprintf(stderr, " -c : ignore warnings\n"); (void) fprintf(stderr, " -v : verbose output\n"); (void) fprintf(stderr, " -U : ignore undefined symbols\n"); (void) fprintf(stderr, " -f charmap : use given charmap file\n"); (void) fprintf(stderr, " -u encoding : assume encoding\n"); (void) fprintf(stderr, " -w widths : use screen widths file\n"); (void) fprintf(stderr, " -i locsrc : source file for locale\n"); exit(4); } int main(int argc, char **argv) { int c; char *lfname = NULL; char *cfname = NULL; char *wfname = NULL; DIR *dir; init_charmap(); init_collate(); init_ctype(); init_messages(); init_monetary(); init_numeric(); init_time(); yydebug = 0; (void) setlocale(LC_ALL, ""); while ((c = getopt(argc, argv, "w:i:cf:u:vUD")) != -1) { switch (c) { case 'D': bsd = 1; break; case 'v': verbose++; break; case 'i': lfname = optarg; break; case 'u': set_wide_encoding(optarg); break; case 'f': cfname = optarg; break; case 'U': undefok++; break; case 'c': warnok++; break; case 'w': wfname = optarg; break; case '?': usage(); break; } } if ((argc - 1) != (optind)) { usage(); } locname = argv[argc - 1]; if (verbose) { (void) printf("Processing locale %s.\n", locname); } if (cfname) { if (verbose) (void) printf("Loading charmap %s.\n", cfname); reset_scanner(cfname); (void) yyparse(); } if (wfname) { if (verbose) (void) printf("Loading widths %s.\n", wfname); reset_scanner(wfname); (void) yyparse(); } if (verbose) { (void) printf("Loading POSIX portable characters.\n"); } add_charmap_posix(); if (lfname) { reset_scanner(lfname); } else { reset_scanner(NULL); } /* make the directory for the locale if not already present */ if (!bsd) { while ((dir = opendir(locname)) == NULL) { if ((errno != ENOENT) || (mkdir(locname, 0755) < 0)) { - errf(strerror(errno)); + errf("%s", strerror(errno)); } } (void) closedir(dir); (void) mkdir(dirname(category_file()), 0755); } (void) yyparse(); if (verbose) { (void) printf("All done.\n"); } return (warnings ? 1 : 0); } Index: head/usr.bin/localedef/localedef.h =================================================================== --- head/usr.bin/localedef/localedef.h (revision 318277) +++ head/usr.bin/localedef/localedef.h (revision 318278) @@ -1,172 +1,173 @@ /*- * Copyright 2010 Nexenta Systems, Inc. All rights reserved. * Copyright 2015 John Marino * * This source code is derived from the illumos localedef command, and * provided under BSD-style license terms by Nexenta Systems, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ /* * POSIX localedef. */ /* Common header files. */ +#include #include #include #include #include extern int com_char; extern int esc_char; extern int mb_cur_max; extern int mb_cur_min; extern int last_kw; extern int verbose; extern int yydebug; extern int lineno; extern int undefok; /* mostly ignore undefined symbols */ extern int warnok; extern int warnings; int yylex(void); void yyerror(const char *); -void errf(const char *, ...); -void warn(const char *, ...); +_Noreturn void errf(const char *, ...) __printflike(1, 2); +void warn(const char *, ...) __printflike(1, 2); int putl_category(const char *, FILE *); int wr_category(void *, size_t, FILE *); FILE *open_category(void); void close_category(FILE *); void copy_category(char *); const char *category_name(void); int get_category(void); int get_symbol(void); int get_escaped(int); int get_wide(void); void reset_scanner(const char *); void scan_to_eol(void); void add_wcs(wchar_t); void add_tok(int); wchar_t *get_wcs(void); /* charmap.c - CHARMAP handling */ void init_charmap(void); void add_charmap(const char *, int); void add_charmap_undefined(char *); void add_charmap_posix(void); void add_charmap_range(char *, char *, int); void add_charmap_char(const char *name, int val); int lookup_charmap(const char *, wchar_t *); int check_charmap_undefined(char *); int check_charmap(wchar_t); /* collate.o - LC_COLLATE handling */ typedef struct collelem collelem_t; typedef struct collsym collsym_t; void init_collate(void); void define_collsym(char *); void define_collelem(char *, wchar_t *); void add_order_directive(void); void add_order_bit(int); void dump_collate(void); collsym_t *lookup_collsym(char *); collelem_t *lookup_collelem(char *); void start_order_collelem(collelem_t *); void start_order_undefined(void); void start_order_symbol(char *); void start_order_char(wchar_t); void start_order_ellipsis(void); void end_order_collsym(collsym_t *); void end_order(void); void add_weight(int32_t, int); void add_weights(int32_t *); void add_weight_num(int); void add_order_collelem(collelem_t *); void add_order_collsym(collsym_t *); void add_order_char(wchar_t); void add_order_ignore(void); void add_order_ellipsis(void); void add_order_symbol(char *); void add_order_subst(void); void add_subst_char(wchar_t); void add_subst_collsym(collsym_t *); void add_subst_collelem(collelem_t *); void add_subst_symbol(char *); int32_t get_weight(int32_t, int); wchar_t * wsncpy(wchar_t *, const wchar_t *, size_t); /* ctype.c - LC_CTYPE handling */ void init_ctype(void); void add_ctype(int); void add_ctype_range(wchar_t); void add_width(int, int); void add_width_range(int, int, int); void add_caseconv(int, int); void dump_ctype(void); /* messages.c - LC_MESSAGES handling */ void init_messages(void); void add_message(wchar_t *); void dump_messages(void); /* monetary.c - LC_MONETARY handling */ void init_monetary(void); void add_monetary_str(wchar_t *); void add_monetary_num(int); void reset_monetary_group(void); void add_monetary_group(int); void dump_monetary(void); /* numeric.c - LC_NUMERIC handling */ void init_numeric(void); void add_numeric_str(wchar_t *); void reset_numeric_group(void); void add_numeric_group(int); void dump_numeric(void); /* time.c - LC_TIME handling */ void init_time(void); void add_time_str(wchar_t *); void reset_time_list(void); void add_time_list(wchar_t *); void check_time_list(void); void dump_time(void); /* wide.c - Wide character handling. */ int to_wide(wchar_t *, const char *); int to_mbs(char *, wchar_t); int to_mb(char *, wchar_t); char *to_mb_string(const wchar_t *); void set_wide_encoding(const char *); void werr(const char *, ...); const char *get_wide_encoding(void); int max_wide(void); //#define _(x) gettext(x) #define INTERR fprintf(stderr,"internal fault (%s:%d)", __FILE__, __LINE__) Index: head/usr.bin/localedef/wide.c =================================================================== --- head/usr.bin/localedef/wide.c (revision 318277) +++ head/usr.bin/localedef/wide.c (revision 318278) @@ -1,666 +1,666 @@ /*- * Copyright 2011 Nexenta Systems, Inc. All rights reserved. * Copyright 2012 Garrett D'Amore All rights reserved. * Copyright 2015 John Marino * * This source code is derived from the illumos localedef command, and * provided under BSD-style license terms by Nexenta Systems, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * The functions in this file convert from the standard multibyte forms * to the wide character forms used internally by libc. Unfortunately, * this approach means that we need a method for each and every encoding. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include "localedef.h" static int towide_none(wchar_t *, const char *, unsigned); static int towide_utf8(wchar_t *, const char *, unsigned); static int towide_big5(wchar_t *, const char *, unsigned); static int towide_gbk(wchar_t *, const char *, unsigned); static int towide_gb2312(wchar_t *, const char *, unsigned); static int towide_gb18030(wchar_t *, const char *, unsigned); static int towide_mskanji(wchar_t *, const char *, unsigned); static int towide_euccn(wchar_t *, const char *, unsigned); static int towide_eucjp(wchar_t *, const char *, unsigned); static int towide_euckr(wchar_t *, const char *, unsigned); static int towide_euctw(wchar_t *, const char *, unsigned); static int tomb_none(char *, wchar_t); static int tomb_utf8(char *, wchar_t); static int tomb_mbs(char *, wchar_t); static int (*_towide)(wchar_t *, const char *, unsigned) = towide_none; static int (*_tomb)(char *, wchar_t) = tomb_none; static char _encoding_buffer[20] = {'N','O','N','E'}; static const char *_encoding = _encoding_buffer; static int _nbits = 7; /* * Table of supported encodings. We only bother to list the multibyte * encodings here, because single byte locales are handed by "NONE". */ static struct { const char *name; /* the name that the underlying libc implemenation uses */ const char *cname; /* the maximum number of bits required for priorities */ int nbits; int (*towide)(wchar_t *, const char *, unsigned); int (*tomb)(char *, wchar_t); } mb_encodings[] = { /* * UTF8 values max out at 0x1fffff (although in theory there could * be later extensions, but it won't happen.) This means we only need * 21 bits to be able to encode the entire range of priorities. */ { "UTF-8", "UTF-8", 21, towide_utf8, tomb_utf8 }, { "UTF8", "UTF-8", 21, towide_utf8, tomb_utf8 }, { "utf8", "UTF-8", 21, towide_utf8, tomb_utf8 }, { "utf-8", "UTF-8", 21, towide_utf8, tomb_utf8 }, { "EUC-CN", "EUC-CN", 16, towide_euccn, tomb_mbs }, { "eucCN", "EUC-CN", 16, towide_euccn, tomb_mbs }, /* * Because the 3-byte form of EUC-JP use the same leading byte, * only 17 bits required to provide unique priorities. (The low * bit of that first byte is set.) By setting this value low, * we can get by with only 3 bytes in the strxfrm expansion. */ { "EUC-JP", "EUC-JP", 17, towide_eucjp, tomb_mbs }, { "eucJP", "EUC-JP", 17, towide_eucjp, tomb_mbs }, { "EUC-KR", "EUC-KR", 16, towide_euckr, tomb_mbs }, { "eucKR", "EUC-KR", 16, towide_euckr, tomb_mbs }, /* * EUC-TW uses 2 bytes most of the time, but 4 bytes if the * high order byte is 0x8E. However, with 4 byte encodings, * the third byte will be A0-B0. So we only need to consider * the lower order 24 bits for collation. */ { "EUC-TW", "EUC-TW", 24, towide_euctw, tomb_mbs }, { "eucTW", "EUC-TW", 24, towide_euctw, tomb_mbs }, { "MS_Kanji", "MSKanji", 16, towide_mskanji, tomb_mbs }, { "MSKanji", "MSKanji", 16, towide_mskanji, tomb_mbs }, { "PCK", "MSKanji", 16, towide_mskanji, tomb_mbs }, { "SJIS", "MSKanji", 16, towide_mskanji, tomb_mbs }, { "Shift_JIS", "MSKanji", 16, towide_mskanji, tomb_mbs }, { "BIG5", "BIG5", 16, towide_big5, tomb_mbs }, { "big5", "BIG5", 16, towide_big5, tomb_mbs }, { "Big5", "BIG5", 16, towide_big5, tomb_mbs }, { "GBK", "GBK", 16, towide_gbk, tomb_mbs }, /* * GB18030 can get away with just 31 bits. This is because the * high order bit is always set for 4 byte values, and the * at least one of the other bits in that 4 byte value will * be non-zero. */ { "GB18030", "GB18030", 31, towide_gb18030, tomb_mbs }, /* * This should probably be an aliase for euc-cn, or vice versa. */ { "GB2312", "GB2312", 16, towide_gb2312, tomb_mbs }, { NULL, NULL, 0, 0, 0 }, }; static char * show_mb(const char *mb) { static char buf[64]; /* ASCII stuff we just print */ if (isascii(*mb) && isgraph(*mb)) { buf[0] = *mb; buf[1] = 0; return (buf); } buf[0] = 0; while (*mb != 0) { char scr[8]; (void) snprintf(scr, sizeof (scr), "\\x%02x", *mb); (void) strlcat(buf, scr, sizeof (buf)); mb++; } return (buf); } static char *widemsg; void werr(const char *fmt, ...) { char *msg; va_list va; va_start(va, fmt); (void) vasprintf(&msg, fmt, va); va_end(va); free(widemsg); widemsg = msg; } /* * This is used for 8-bit encodings. */ int towide_none(wchar_t *c, const char *mb, unsigned n __unused) { if (mb_cur_max != 1) { werr("invalid or unsupported multibyte locale"); return (-1); } *c = (uint8_t)*mb; return (1); } int tomb_none(char *mb, wchar_t wc) { if (mb_cur_max != 1) { werr("invalid or unsupported multibyte locale"); return (-1); } *(uint8_t *)mb = (wc & 0xff); mb[1] = 0; return (1); } /* * UTF-8 stores wide characters in UTF-32 form. */ int towide_utf8(wchar_t *wc, const char *mb, unsigned n) { wchar_t c; int nb; wchar_t lv; /* lowest legal value */ int i; const uint8_t *s = (const uint8_t *)mb; c = *s; if ((c & 0x80) == 0) { /* 7-bit ASCII */ *wc = c; return (1); } else if ((c & 0xe0) == 0xc0) { /* u80-u7ff - two bytes encoded */ nb = 2; lv = 0x80; c &= ~0xe0; } else if ((c & 0xf0) == 0xe0) { /* u800-uffff - three bytes encoded */ nb = 3; lv = 0x800; c &= ~0xf0; } else if ((c & 0xf8) == 0xf0) { /* u1000-u1fffff - four bytes encoded */ nb = 4; lv = 0x1000; c &= ~0xf8; } else { /* 5 and 6 byte encodings are not legal unicode */ werr("utf8 encoding too large (%s)", show_mb(mb)); return (-1); } if (nb > (int)n) { werr("incomplete utf8 sequence (%s)", show_mb(mb)); return (-1); } for (i = 1; i < nb; i++) { if (((s[i]) & 0xc0) != 0x80) { werr("illegal utf8 byte (%x)", s[i]); return (-1); } c <<= 6; c |= (s[i] & 0x3f); } if (c < lv) { werr("illegal redundant utf8 encoding (%s)", show_mb(mb)); return (-1); } *wc = c; return (nb); } int tomb_utf8(char *mb, wchar_t wc) { uint8_t *s = (uint8_t *)mb; uint8_t msk; int cnt; int i; if (wc <= 0x7f) { s[0] = wc & 0x7f; s[1] = 0; return (1); } if (wc <= 0x7ff) { cnt = 2; msk = 0xc0; } else if (wc <= 0xffff) { cnt = 3; msk = 0xe0; } else if (wc <= 0x1fffff) { cnt = 4; msk = 0xf0; } else { werr("illegal uf8 char (%x)", wc); return (-1); } for (i = cnt - 1; i; i--) { s[i] = (wc & 0x3f) | 0x80; wc >>= 6; } s[0] = (msk) | wc; s[cnt] = 0; return (cnt); } /* * Several encodings share a simplistic dual byte encoding. In these * forms, they all indicate that a two byte sequence is to be used if * the first byte has its high bit set. They all store this simple * encoding as a 16-bit value, although a great many of the possible * code points are not used in most character sets. This gives a possible * set of just over 32,000 valid code points. * * 0x00 - 0x7f - 1 byte encoding * 0x80 - 0x7fff - illegal * 0x8000 - 0xffff - 2 byte encoding */ static int towide_dbcs(wchar_t *wc, const char *mb, unsigned n) { wchar_t c; c = *(const uint8_t *)mb; if ((c & 0x80) == 0) { /* 7-bit */ *wc = c; return (1); } if (n < 2) { werr("incomplete character sequence (%s)", show_mb(mb)); return (-1); } /* Store both bytes as a single 16-bit wide. */ c <<= 8; c |= (uint8_t)(mb[1]); *wc = c; return (2); } /* * Most multibyte locales just convert the wide character to the multibyte * form by stripping leading null bytes, and writing the 32-bit quantity * in big-endian order. */ int tomb_mbs(char *mb, wchar_t wc) { uint8_t *s = (uint8_t *)mb; int n = 0, c; if ((wc & 0xff000000U) != 0) { n = 4; } else if ((wc & 0x00ff0000U) != 0) { n = 3; } else if ((wc & 0x0000ff00U) != 0) { n = 2; } else { n = 1; } c = n; while (n) { n--; s[n] = wc & 0xff; wc >>= 8; } /* ensure null termination */ s[c] = 0; return (c); } /* * big5 is a simple dual byte character set. */ int towide_big5(wchar_t *wc, const char *mb, unsigned n) { return (towide_dbcs(wc, mb, n)); } /* * GBK encodes wides in the same way that big5 does, the high order * bit of the first byte indicates a double byte character. */ int towide_gbk(wchar_t *wc, const char *mb, unsigned n) { return (towide_dbcs(wc, mb, n)); } /* * GB2312 is another DBCS. Its cleaner than others in that the second * byte does not encode ASCII, but it supports characters. */ int towide_gb2312(wchar_t *wc, const char *mb, unsigned n) { return (towide_dbcs(wc, mb, n)); } /* * GB18030. This encodes as 8, 16, or 32-bits. * 7-bit values are in 1 byte, 4 byte sequences are used when * the second byte encodes 0x30-39 and all other sequences are 2 bytes. */ int towide_gb18030(wchar_t *wc, const char *mb, unsigned n) { wchar_t c; c = *(const uint8_t *)mb; if ((c & 0x80) == 0) { /* 7-bit */ *wc = c; return (1); } if (n < 2) { werr("incomplete character sequence (%s)", show_mb(mb)); return (-1); } /* pull in the second byte */ c <<= 8; c |= (uint8_t)(mb[1]); if (((c & 0xff) >= 0x30) && ((c & 0xff) <= 0x39)) { if (n < 4) { werr("incomplete 4-byte character sequence (%s)", show_mb(mb)); return (-1); } c <<= 8; c |= (uint8_t)(mb[2]); c <<= 8; c |= (uint8_t)(mb[3]); *wc = c; return (4); } *wc = c; return (2); } /* * MS-Kanji (aka SJIS) is almost a clean DBCS like the others, but it * also has a range of single byte characters above 0x80. (0xa1-0xdf). */ int towide_mskanji(wchar_t *wc, const char *mb, unsigned n) { wchar_t c; c = *(const uint8_t *)mb; if ((c < 0x80) || ((c > 0xa0) && (c < 0xe0))) { /* 7-bit */ *wc = c; return (1); } if (n < 2) { werr("incomplete character sequence (%s)", show_mb(mb)); return (-1); } /* Store both bytes as a single 16-bit wide. */ c <<= 8; c |= (uint8_t)(mb[1]); *wc = c; return (2); } /* * EUC forms. EUC encodings are "variable". FreeBSD carries some additional * variable data to encode these, but we're going to treat each as independent * instead. Its the only way we can sensibly move forward. * * Note that the way in which the different EUC forms vary is how wide * CS2 and CS3 are and what the first byte of them is. */ static int towide_euc_impl(wchar_t *wc, const char *mb, unsigned n, uint8_t cs2, uint8_t cs2width, uint8_t cs3, uint8_t cs3width) { int i; int width = 2; wchar_t c; c = *(const uint8_t *)mb; /* * All variations of EUC encode 7-bit ASCII as one byte, and use * additional bytes for more than that. */ if ((c & 0x80) == 0) { /* 7-bit */ *wc = c; return (1); } /* * All EUC variants reserve 0xa1-0xff to identify CS1, which * is always two bytes wide. Note that unused CS will be zero, * and that cannot be true because we know that the high order * bit must be set. */ if (c >= 0xa1) { width = 2; } else if (c == cs2) { width = cs2width; } else if (c == cs3) { width = cs3width; } if ((int)n < width) { werr("incomplete character sequence (%s)", show_mb(mb)); return (-1); } for (i = 1; i < width; i++) { /* pull in the next byte */ c <<= 8; c |= (uint8_t)(mb[i]); } *wc = c; return (width); } /* * EUC-CN encodes as follows: * * Code set 0 (ASCII): 0x21-0x7E * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE * Code set 2: unused * Code set 3: unused */ int towide_euccn(wchar_t *wc, const char *mb, unsigned n) { return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0)); } /* * EUC-JP encodes as follows: * * Code set 0 (ASCII or JIS X 0201-1976 Roman): 0x21-0x7E * Code set 1 (JIS X 0208): 0xA1A1-0xFEFE * Code set 2 (half-width katakana): 0x8EA1-0x8EDF * Code set 3 (JIS X 0212-1990): 0x8FA1A1-0x8FFEFE */ int towide_eucjp(wchar_t *wc, const char *mb, unsigned n) { return (towide_euc_impl(wc, mb, n, 0x8e, 2, 0x8f, 3)); } /* * EUC-KR encodes as follows: * * Code set 0 (ASCII or KS C 5636-1993): 0x21-0x7E * Code set 1 (KS C 5601-1992): 0xA1A1-0xFEFE * Code set 2: unused * Code set 3: unused */ int towide_euckr(wchar_t *wc, const char *mb, unsigned n) { return (towide_euc_impl(wc, mb, n, 0, 0, 0, 0)); } /* * EUC-TW encodes as follows: * * Code set 0 (ASCII): 0x21-0x7E * Code set 1 (CNS 11643-1992 Plane 1): 0xA1A1-0xFEFE * Code set 2 (CNS 11643-1992 Planes 1-16): 0x8EA1A1A1-0x8EB0FEFE * Code set 3: unused */ int towide_euctw(wchar_t *wc, const char *mb, unsigned n) { return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0)); } /* * Public entry points. */ int to_wide(wchar_t *wc, const char *mb) { /* this won't fail hard */ return (_towide(wc, mb, strlen(mb))); } int to_mb(char *mb, wchar_t wc) { int rv; if ((rv = _tomb(mb, wc)) < 0) { - errf(widemsg); + warn("%s", widemsg); free(widemsg); widemsg = NULL; } return (rv); } char * to_mb_string(const wchar_t *wcs) { char *mbs; char *ptr; int len; mbs = malloc((wcslen(wcs) * mb_cur_max) + 1); if (mbs == NULL) { - errf("out of memory"); + warn("out of memory"); return (NULL); } ptr = mbs; while (*wcs) { if ((len = to_mb(ptr, *wcs)) < 0) { INTERR; free(mbs); return (NULL); } wcs++; ptr += len; } *ptr = 0; return (mbs); } void set_wide_encoding(const char *encoding) { int i; _towide = towide_none; _tomb = tomb_none; _nbits = 8; snprintf(_encoding_buffer, sizeof(_encoding_buffer), "NONE:%s", encoding); for (i = 0; mb_encodings[i].name; i++) { if (strcasecmp(encoding, mb_encodings[i].name) == 0) { _towide = mb_encodings[i].towide; _tomb = mb_encodings[i].tomb; _encoding = mb_encodings[i].cname; _nbits = mb_encodings[i].nbits; break; } } } const char * get_wide_encoding(void) { return (_encoding); } int max_wide(void) { return ((int)((1U << _nbits) - 1)); }