Page MenuHomeFreeBSD
Files F173197120

D59213.diff
No OneTemporary
Actions

D59213.diff
View Options

diff --git a/include/uchar.h b/include/uchar.h
--- a/include/uchar.h
+++ b/include/uchar.h
@@ -32,7 +32,7 @@
#include <sys/cdefs.h>
#include <sys/_types.h>
-#if __ISO_C_VISIBLE >= 2023 && !defined(_CHAR8_T_DECLARED)
+#ifndef _CHAR8_T_DECLARED
typedef unsigned char char8_t;
#define _CHAR8_T_DECLARED
#endif
@@ -58,6 +58,16 @@
#endif
__BEGIN_DECLS
+/*
+ * TODO: libc++ sets _LIBCPP_HAS_C8RTOMB_MBRTOC8 only for glibc at the time,
+ * so std::mbrtoc8 / std::c8rtomb are not provided by <cuchar> yet and needs
+ * us to submit a PR to libc++ to add them.
+ */
+#if !defined(__cplusplus) || defined(__cpp_char8_t)
+size_t c8rtomb(char * __restrict, char8_t, mbstate_t * __restrict);
+size_t mbrtoc8(char8_t * __restrict, const char * __restrict, size_t,
+ mbstate_t * __restrict);
+#endif
size_t c16rtomb(char * __restrict, char16_t, mbstate_t * __restrict);
size_t c32rtomb(char * __restrict, char32_t, mbstate_t * __restrict);
size_t mbrtoc16(char16_t * __restrict, const char * __restrict, size_t,
diff --git a/include/xlocale/_uchar.h b/include/xlocale/_uchar.h
--- a/include/xlocale/_uchar.h
+++ b/include/xlocale/_uchar.h
@@ -34,6 +34,12 @@
#ifndef _XLOCALE_UCHAR_H_
#define _XLOCALE_UCHAR_H_
+#if !defined(__cplusplus) || defined(__cpp_char8_t)
+size_t c8rtomb_l(char * __restrict, char8_t, mbstate_t * __restrict,
+ locale_t);
+size_t mbrtoc8_l(char8_t * __restrict, const char * __restrict, size_t,
+ mbstate_t * __restrict, locale_t);
+#endif
size_t c16rtomb_l(char * __restrict, char16_t, mbstate_t * __restrict,
locale_t);
size_t c32rtomb_l(char * __restrict, char32_t, mbstate_t * __restrict,
diff --git a/lib/libc/locale/Makefile.inc b/lib/libc/locale/Makefile.inc
--- a/lib/libc/locale/Makefile.inc
+++ b/lib/libc/locale/Makefile.inc
@@ -21,9 +21,11 @@
xlocale.c
.if ${MK_ICONV} != "no"
-SRCS+= c16rtomb_iconv.c c32rtomb_iconv.c mbrtoc16_iconv.c mbrtoc32_iconv.c
+SRCS+= c8rtomb_iconv.c c16rtomb_iconv.c c32rtomb_iconv.c \
+ mbrtoc8_iconv.c mbrtoc16_iconv.c mbrtoc32_iconv.c
.else
-SRCS+= c16rtomb.c c32rtomb.c mbrtoc16.c mbrtoc32.c
+SRCS+= c8rtomb.c c16rtomb.c c32rtomb.c \
+ mbrtoc8.c mbrtoc16.c mbrtoc32.c
.endif
SYM_MAPS+=${LIBC_SRCTOP}/locale/Symbol.map
@@ -90,9 +92,9 @@
MLINKS+=isxdigit.3 ishexnumber.3 isxdigit.3 isxdigit_l.3 \
isxdigit.3 ishexnumber_l.3
MLINKS+=localeconv.3 localeconv_l.3
-MLINKS+=mbrtowc.3 mbrtoc16.3 mbrtowc.3 mbrtoc32.3
+MLINKS+=mbrtowc.3 mbrtoc8.3 mbrtowc.3 mbrtoc16.3 mbrtowc.3 mbrtoc32.3
MLINKS+=mbsrtowcs.3 mbsnrtowcs.3
-MLINKS+=wcrtomb.3 c16rtomb.3 wcrtomb.3 c32rtomb.3
+MLINKS+=wcrtomb.3 c8rtomb.3 wcrtomb.3 c16rtomb.3 wcrtomb.3 c32rtomb.3
MLINKS+=wcsrtombs.3 wcsnrtombs.3
MLINKS+=wcstod.3 wcstof.3 wcstod.3 wcstold.3
MLINKS+=wcstol.3 wcstoul.3 wcstol.3 wcstoll.3 wcstol.3 wcstoull.3 \
diff --git a/lib/libc/locale/Symbol.map b/lib/libc/locale/Symbol.map
--- a/lib/libc/locale/Symbol.map
+++ b/lib/libc/locale/Symbol.map
@@ -205,6 +205,13 @@
mbrtoc32_l;
};
+FBSD_1.9 {
+ c8rtomb;
+ c8rtomb_l;
+ mbrtoc8;
+ mbrtoc8_l;
+};
+
FBSDprivate_1.0 {
_PathLocale;
__detect_path_locale;
diff --git a/lib/libc/locale/c8rtomb.c b/lib/libc/locale/c8rtomb.c
new file mode 100644
--- /dev/null
+++ b/lib/libc/locale/c8rtomb.c
@@ -0,0 +1,113 @@
+/*
+ * Copyright (c) 2026 Faraz Vahedi <kfv@FreeBSD.org>
+ * Copyright (c) 2026 Robert Clausecker <fuz@FreeBSD.org>
+ * Copyright (c) 2008-2010 Bjoern Hoehrmann <bjoern@hoehrmann.de>
+ * See http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ for details.
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#include <errno.h>
+#include <limits.h>
+#include <stdint.h>
+#include <uchar.h>
+#include "mblocal.h"
+
+/*
+ * UTF-8 accumulation lives in the last bytes of mbstate_t so that the
+ * c32rtomb conversion state, stored at the front, remains a complete
+ * mbstate_t (C23 §7.30.1p2: char8_t sequences are UTF-8).
+ */
+typedef struct {
+ unsigned state;
+ char32_t codep;
+} _Char8State;
+
+_Static_assert(sizeof(_Char8State) < sizeof(mbstate_t),
+ "UTF-8 extra state must fit in mbstate_t");
+_Static_assert((sizeof(mbstate_t) - sizeof(_Char8State)) %
+ _Alignof(_Char8State) == 0,
+ "UTF-8 extra state must be aligned at the end of mbstate_t");
+
+static _Char8State *
+c8rtomb_extra(mbstate_t *ps)
+{
+ return ((_Char8State *)((char *)ps + sizeof(*ps) -
+ sizeof(_Char8State)));
+}
+
+enum {
+ ACCEPT = 0, /* complete UTF-8 sequence */
+ REJECT = 12, /* illegal sequence */
+};
+
+static const unsigned char utf8d[] = {
+ /*
+ * The first part of the table maps bytes to character classes that
+ * to reduce the size of the transition table and create bitmasks.
+ */
+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
+ 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+ 8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
+ 10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,
+
+ /*
+ * The second part is a transition table that maps a combination
+ * of a state of the automaton and a character class to a state.
+ */
+ 0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12,
+ 12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12,
+ 12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12,
+ 12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12,
+ 12,36,12,12,12,12,12,12,12,12,12,12,
+};
+
+size_t
+c8rtomb_l(char * __restrict s, char8_t c8, mbstate_t * __restrict ps,
+ locale_t locale)
+{
+ _Char8State *ep;
+ unsigned type;
+ char nbuf[MB_LEN_MAX];
+
+ FIX_LOCALE(locale);
+ if (ps == NULL)
+ ps = &(XLOCALE_CTYPE(locale)->c8rtomb);
+ ep = c8rtomb_extra(ps);
+
+ /*
+ * C23 §7.30.1.2p2: if s is null, the function is equivalent to the
+ * call c8rtomb(buf, u8'\0', ps) with the current conversion state.
+ * Incomplete UTF-8 sequence followed by U+0000 is an encoding error.
+ */
+ if (s == NULL) {
+ c8 = 0;
+ s = nbuf;
+ }
+
+ type = utf8d[c8];
+ ep->codep = (ep->state != ACCEPT) ?
+ (c8 & 0x3f) | (ep->codep << 6) : (0xff >> type) & c8;
+ ep->state = utf8d[256 + ep->state + type];
+
+ switch (ep->state) {
+ case ACCEPT:
+ return (c32rtomb_l(s, ep->codep, ps, locale));
+ case REJECT:
+ ep->state = ACCEPT;
+ errno = EILSEQ;
+ return ((size_t)-1);
+ default:
+ return (0);
+ }
+}
+
+size_t
+c8rtomb(char * __restrict s, char8_t c8, mbstate_t * __restrict ps)
+{
+ return (c8rtomb_l(s, c8, ps, __get_locale()));
+}
diff --git a/lib/libc/locale/c8rtomb_iconv.c b/lib/libc/locale/c8rtomb_iconv.c
new file mode 100644
--- /dev/null
+++ b/lib/libc/locale/c8rtomb_iconv.c
@@ -0,0 +1,7 @@
+#define charXX_t char8_t
+#define cXXrtomb c8rtomb
+#define cXXrtomb_l c8rtomb_l
+#define SRCBUF_LEN 6
+#define UTF_XX_INTERNAL "UTF-8"
+
+#include "cXXrtomb_iconv.h"
diff --git a/lib/libc/locale/mblocal.h b/lib/libc/locale/mblocal.h
--- a/lib/libc/locale/mblocal.h
+++ b/lib/libc/locale/mblocal.h
@@ -62,6 +62,8 @@
__mbstate_t mblen;
/** Persistent state used by mbrlen() calls. */
__mbstate_t mbrlen;
+ /** Persistent state used by mbrtoc8() calls. */
+ __mbstate_t mbrtoc8;
/** Persistent state used by mbrtoc16() calls. */
__mbstate_t mbrtoc16;
/** Persistent state used by mbrtoc32() calls. */
@@ -74,6 +76,8 @@
__mbstate_t mbsrtowcs;
/** Persistent state used by mbtowc() calls. */
__mbstate_t mbtowc;
+ /** Persistent state used by c8rtomb() calls. */
+ __mbstate_t c8rtomb;
/** Persistent state used by c16rtomb() calls. */
__mbstate_t c16rtomb;
/** Persistent state used by c32rtomb() calls. */
diff --git a/lib/libc/locale/mbrtoc8.c b/lib/libc/locale/mbrtoc8.c
new file mode 100644
--- /dev/null
+++ b/lib/libc/locale/mbrtoc8.c
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2026 Faraz Vahedi <kfv@FreeBSD.org>
+ * Copyright (c) 2026 Robert Clausecker <fuz@FreeBSD.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <errno.h>
+#include <uchar.h>
+#include "mblocal.h"
+
+typedef struct {
+ char32_t c32; /* character being returned */
+ int pending; /* trailing code units still to return */
+ mbstate_t c32_mbstate;
+} _Char8State;
+
+size_t
+mbrtoc8_l(char8_t * __restrict pc8, const char * __restrict s, size_t n,
+ mbstate_t * __restrict ps, locale_t locale)
+{
+ _Char8State *cs;
+ char32_t c32;
+ char8_t c8;
+ ssize_t len;
+
+ FIX_LOCALE(locale);
+ if (ps == NULL)
+ ps = &(XLOCALE_CTYPE(locale)->mbrtoc8);
+ cs = (_Char8State *)ps;
+
+ /*
+ * If s is a null pointer, the value of parameter pc8 is also
+ * ignored. Pending code units are discarded, as mbrtoc16()
+ * does with its trail surrogate.
+ */
+ if (s == NULL) {
+ cs->pending = 0;
+ return (mbrtoc32_l(NULL, s, n, &cs->c32_mbstate, locale));
+ }
+
+ /* Return the next trailing code unit from the previous invocation. */
+ if (cs->pending > 0) {
+ cs->pending--;
+ if (pc8 != NULL)
+ *pc8 = 0x80 | ((cs->c32 >> (6 * cs->pending)) & 0x3f);
+ return ((size_t)-3);
+ }
+
+ len = mbrtoc32_l(&c32, s, n, &cs->c32_mbstate, locale);
+ if (len >= 0) {
+ /* Unicode Standard 5.0, D90: ill-formed characters. */
+ if ((c32 >= 0xd800 && c32 <= 0xdfff) || c32 > 0x10ffff) {
+ errno = EILSEQ;
+ return ((size_t)-1);
+ }
+ /* Return the leading code unit, keep the rest for later. */
+ if (c32 < 0x80) {
+ c8 = c32;
+ cs->pending = 0;
+ } else if (c32 < 0x800) {
+ c8 = 0xc0 | (c32 >> 6);
+ cs->pending = 1;
+ } else if (c32 < 0x10000) {
+ c8 = 0xe0 | (c32 >> 12);
+ cs->pending = 2;
+ } else {
+ c8 = 0xf0 | (c32 >> 18);
+ cs->pending = 3;
+ }
+ cs->c32 = c32;
+ if (pc8 != NULL)
+ *pc8 = c8;
+ }
+ return (len);
+}
+
+size_t
+mbrtoc8(char8_t * __restrict pc8, const char * __restrict s, size_t n,
+ mbstate_t * __restrict ps)
+{
+ return (mbrtoc8_l(pc8, s, n, ps, __get_locale()));
+}
diff --git a/lib/libc/locale/mbrtoc8_iconv.c b/lib/libc/locale/mbrtoc8_iconv.c
new file mode 100644
--- /dev/null
+++ b/lib/libc/locale/mbrtoc8_iconv.c
@@ -0,0 +1,7 @@
+#define charXX_t char8_t
+#define mbrtocXX mbrtoc8
+#define mbrtocXX_l mbrtoc8_l
+#define DSTBUF_LEN 6
+#define UTF_XX_INTERNAL "UTF-8"
+
+#include "mbrtocXX_iconv.h"
diff --git a/lib/libc/locale/mbrtowc.3 b/lib/libc/locale/mbrtowc.3
--- a/lib/libc/locale/mbrtowc.3
+++ b/lib/libc/locale/mbrtowc.3
@@ -22,11 +22,12 @@
.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
.\" SUCH DAMAGE.
.\"
-.Dd May 21, 2013
+.Dd August 30, 2026
.Dt MBRTOWC 3
.Os
.Sh NAME
.Nm mbrtowc ,
+.Nm mbrtoc8 ,
.Nm mbrtoc16 ,
.Nm mbrtoc32
.Nd "convert a character to a wide-character code (restartable)"
@@ -41,6 +42,11 @@
.Fc
.In uchar.h
.Ft size_t
+.Fo mbrtoc8
+.Fa "char8_t * restrict pc" "const char * restrict s" "size_t n"
+.Fa "mbstate_t * restrict ps"
+.Fc
+.Ft size_t
.Fo mbrtoc16
.Fa "char16_t * restrict pc" "const char * restrict s" "size_t n"
.Fa "mbstate_t * restrict ps"
@@ -53,6 +59,7 @@
.Sh DESCRIPTION
The
.Fn mbrtowc ,
+.Fn mbrtoc8 ,
.Fn mbrtoc16
and
.Fn mbrtoc32
@@ -70,6 +77,7 @@
.Fa s
is stored in the
.Vt wchar_t ,
+.Vt char8_t ,
.Vt char16_t
or
.Vt char32_t
@@ -103,14 +111,22 @@
at program startup.
.Pp
As a single
+.Vt char8_t
+or
.Vt char16_t
-is not large enough to represent certain multibyte characters, the
+is not large enough to represent certain Unicode characters, the
+.Fn mbrtoc8
+and
.Fn mbrtoc16
-function may need to be invoked multiple times to convert a single
+functions may need to be invoked multiple times to convert a single
multibyte character sequence.
+The encoding of
+.Vt char8_t
+sequences is UTF-8.
.Sh RETURN VALUES
The
.Fn mbrtowc ,
+.Fn mbrtoc8 ,
.Fn mbrtoc16
and
.Fn mbrtoc32
@@ -142,8 +158,10 @@
.El
.Pp
The
+.Fn mbrtoc8
+and
.Fn mbrtoc16
-function also returns:
+functions also return:
.Bl -tag -width indent
.It Po Vt size_t Pc Ns \-3
The next character resulting from a previous call has been stored.
@@ -152,6 +170,7 @@
.Sh ERRORS
The
.Fn mbrtowc ,
+.Fn mbrtoc8 ,
.Fn mbrtoc16
and
.Fn mbrtoc32
@@ -175,3 +194,7 @@
.Fn mbrtoc32
functions conform to
.St -isoC-2011 .
+The
+.Fn mbrtoc8
+function conforms to
+.St -isoC-2023 .
diff --git a/lib/libc/locale/setrunelocale.c b/lib/libc/locale/setrunelocale.c
--- a/lib/libc/locale/setrunelocale.c
+++ b/lib/libc/locale/setrunelocale.c
@@ -156,10 +156,12 @@
/* Free the old runes if it exists. */
free_runes(saved.runes);
/* Reset the mbstates */
+ memset(&l->c8rtomb, 0, sizeof(l->c8rtomb));
memset(&l->c16rtomb, 0, sizeof(l->c16rtomb));
memset(&l->c32rtomb, 0, sizeof(l->c32rtomb));
memset(&l->mblen, 0, sizeof(l->mblen));
memset(&l->mbrlen, 0, sizeof(l->mbrlen));
+ memset(&l->mbrtoc8, 0, sizeof(l->mbrtoc8));
memset(&l->mbrtoc16, 0, sizeof(l->mbrtoc16));
memset(&l->mbrtoc32, 0, sizeof(l->mbrtoc32));
memset(&l->mbrtowc, 0, sizeof(l->mbrtowc));
diff --git a/lib/libc/locale/wcrtomb.3 b/lib/libc/locale/wcrtomb.3
--- a/lib/libc/locale/wcrtomb.3
+++ b/lib/libc/locale/wcrtomb.3
@@ -22,11 +22,12 @@
.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
.\" SUCH DAMAGE.
.\"
-.Dd May 21, 2013
+.Dd August 30, 2026
.Dt WCRTOMB 3
.Os
.Sh NAME
.Nm wcrtomb ,
+.Nm c8rtomb ,
.Nm c16rtomb ,
.Nm c32rtomb
.Nd "convert a wide-character code to a character (restartable)"
@@ -38,12 +39,15 @@
.Fn wcrtomb "char * restrict s" "wchar_t c" "mbstate_t * restrict ps"
.In uchar.h
.Ft size_t
+.Fn c8rtomb "char * restrict s" "char8_t c" "mbstate_t * restrict ps"
+.Ft size_t
.Fn c16rtomb "char * restrict s" "char16_t c" "mbstate_t * restrict ps"
.Ft size_t
.Fn c32rtomb "char * restrict s" "char32_t c" "mbstate_t * restrict ps"
.Sh DESCRIPTION
The
.Fn wcrtomb ,
+.Fn c8rtomb ,
.Fn c16rtomb
and
.Fn c32rtomb
@@ -79,11 +83,16 @@
object, which is initialized to the initial conversion state
at program startup.
.Pp
-As certain multibyte characters may only be represented by a series of
-16-bit characters, the
+As certain Unicode characters may only be represented by a series of
+UTF-8 or UTF-16 code units, the
+.Fn c8rtomb
+and
.Fn c16rtomb
-may need to invoked multiple times before a multibyte sequence is
+functions may need to be invoked multiple times before a multibyte sequence is
returned.
+The encoding of
+.Vt char8_t
+sequences is UTF-8.
.Sh RETURN VALUES
These functions return the length (in bytes) of the multibyte sequence
needed to represent
@@ -96,6 +105,7 @@
.Sh ERRORS
The
.Fn wcrtomb ,
+.Fn c8rtomb ,
.Fn c16rtomb
and
.Fn c32rtomb
@@ -119,3 +129,7 @@
.Fn c32rtomb
functions conform to
.St -isoC-2011 .
+The
+.Fn c8rtomb
+function conforms to
+.St -isoC-2023 .
diff --git a/lib/libc/tests/locale/Makefile b/lib/libc/tests/locale/Makefile
--- a/lib/libc/tests/locale/Makefile
+++ b/lib/libc/tests/locale/Makefile
@@ -1,10 +1,12 @@
.include <bsd.own.mk>
ATF_TESTS_C+= btowc_test
+ATF_TESTS_C+= c8rtomb_test
ATF_TESTS_C+= c16rtomb_test
ATF_TESTS_C+= iswctype_test
ATF_TESTS_C+= mblen_test
ATF_TESTS_C+= mbrlen_test
+ATF_TESTS_C+= mbrtoc8_test
ATF_TESTS_C+= mbrtoc16_test
ATF_TESTS_C+= mbrtowc_2_test
ATF_TESTS_C+= mbsnrtowcs_2_test
diff --git a/lib/libc/tests/locale/c8rtomb_test.c b/lib/libc/tests/locale/c8rtomb_test.c
new file mode 100644
--- /dev/null
+++ b/lib/libc/tests/locale/c8rtomb_test.c
@@ -0,0 +1,201 @@
+/*
+ * Copyright (c) 2026 Faraz Vahedi <kfv@FreeBSD.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+/*
+ * Test program for c8rtomb() as specified by ISO/IEC 9899:2024, §7.30.1.2.
+ */
+
+#include <errno.h>
+#include <limits.h>
+#include <locale.h>
+#include <stdio.h>
+#include <string.h>
+#include <uchar.h>
+
+#include <atf-c.h>
+
+static void
+require_lc_ctype(const char *locale_name)
+{
+ char *lc_ctype_set;
+
+ lc_ctype_set = setlocale(LC_CTYPE, locale_name);
+ if (lc_ctype_set == NULL)
+ atf_tc_fail("setlocale(LC_CTYPE, \"%s\") failed; errno=%d",
+ locale_name, errno);
+
+ ATF_REQUIRE_STREQ(locale_name, lc_ctype_set);
+}
+
+static mbstate_t s;
+static char buf[MB_LEN_MAX + 1];
+
+/*
+ * Feed an ill-formed UTF-8 sequence one code unit at a time. It may be
+ * rejected at the first offending code unit or only once the whole
+ * sequence has been seen, but it must never be converted.
+ */
+static void
+require_ill_formed(const char8_t *seq, size_t n)
+{
+ size_t i, r;
+
+ memset(&s, 0, sizeof(s));
+ memset(buf, 0xcc, sizeof(buf));
+ r = 0;
+ for (i = 0; i < n && r != (size_t)-1; i++) {
+ errno = 0;
+ r = c8rtomb(buf, seq[i], &s);
+ ATF_REQUIRE_MSG(r == 0 || r == (size_t)-1,
+ "code unit %zu of %zu was converted", i + 1, n);
+ }
+ ATF_REQUIRE_EQ_MSG((size_t)-1, r, "sequence of %zu code units accepted",
+ n);
+ ATF_REQUIRE_EQ(EILSEQ, errno);
+ ATF_REQUIRE_EQ(0xcc, (unsigned char)buf[0]);
+}
+
+static const char8_t overlong_nul[] = { 0xc0, 0x80 };
+static const char8_t utf8_surrogate[] = { 0xed, 0xa0, 0x80 };
+
+ATF_TC_WITHOUT_HEAD(c8rtomb_c_locale_test);
+ATF_TC_BODY(c8rtomb_c_locale_test, tc)
+{
+ require_lc_ctype("C");
+
+ /*
+ * If s is a null pointer, the call is equivalent to
+ * c8rtomb(buf, u8'\0', ps) (C23 §7.30.1.2p2). In the initial
+ * conversion state this writes a null byte and returns 1.
+ */
+ ATF_REQUIRE_EQ(1, c8rtomb(NULL, 0, NULL));
+ ATF_REQUIRE_EQ(1, c8rtomb(NULL, 0x80, NULL));
+
+ /* Null character. */
+ memset(&s, 0, sizeof(s));
+ memset(buf, 0xcc, sizeof(buf));
+ ATF_REQUIRE_EQ(1, c8rtomb(buf, 0, &s));
+ ATF_REQUIRE_EQ(0, (unsigned char)buf[0]);
+ ATF_REQUIRE_EQ(0xcc, (unsigned char)buf[1]);
+
+ /* Latin letter A, internal state. */
+ ATF_REQUIRE_EQ(1, c8rtomb(NULL, 0, NULL));
+ ATF_REQUIRE_EQ(1, c8rtomb(NULL, 'A', NULL));
+
+ /* Latin letter A. */
+ memset(&s, 0, sizeof(s));
+ memset(buf, 0xcc, sizeof(buf));
+ ATF_REQUIRE_EQ(1, c8rtomb(buf, 'A', &s));
+ ATF_REQUIRE_EQ('A', (unsigned char)buf[0]);
+ ATF_REQUIRE_EQ(0xcc, (unsigned char)buf[1]);
+
+ /* Isolated UTF-8 continuation byte. */
+ memset(&s, 0, sizeof(s));
+ memset(buf, 0xcc, sizeof(buf));
+ ATF_REQUIRE_EQ((size_t)-1, c8rtomb(buf, 0x80, &s));
+ ATF_REQUIRE_EQ(EILSEQ, errno);
+ ATF_REQUIRE_EQ(0xcc, (unsigned char)buf[0]);
+
+ /* Unicode character 'Pile of poo' cannot be represented. */
+ memset(&s, 0, sizeof(s));
+ memset(buf, 0xcc, sizeof(buf));
+ ATF_REQUIRE_EQ(0, c8rtomb(buf, 0xf0, &s));
+ ATF_REQUIRE_EQ(0, c8rtomb(buf, 0x9f, &s));
+ ATF_REQUIRE_EQ(0, c8rtomb(buf, 0x92, &s));
+ ATF_REQUIRE_EQ((size_t)-1, c8rtomb(buf, 0xa9, &s));
+ ATF_REQUIRE_EQ(EILSEQ, errno);
+ ATF_REQUIRE_EQ(0xcc, (unsigned char)buf[0]);
+}
+
+ATF_TC_WITHOUT_HEAD(c8rtomb_iso_8859_1_test);
+ATF_TC_BODY(c8rtomb_iso_8859_1_test, tc)
+{
+ require_lc_ctype("en_US.ISO8859-1");
+
+ /* Euro sign cannot be represented. */
+ memset(&s, 0, sizeof(s));
+ memset(buf, 0xcc, sizeof(buf));
+ ATF_REQUIRE_EQ(0, c8rtomb(buf, 0xe2, &s));
+ ATF_REQUIRE_EQ(0, c8rtomb(buf, 0x82, &s));
+ ATF_REQUIRE_EQ((size_t)-1, c8rtomb(buf, 0xac, &s));
+ ATF_REQUIRE_EQ(EILSEQ, errno);
+ ATF_REQUIRE_EQ(0xcc, (unsigned char)buf[0]);
+}
+
+ATF_TC_WITHOUT_HEAD(c8rtomb_iso_8859_15_test);
+ATF_TC_BODY(c8rtomb_iso_8859_15_test, tc)
+{
+ require_lc_ctype("en_US.ISO8859-15");
+
+ /* Euro sign U+20AC. */
+ memset(&s, 0, sizeof(s));
+ memset(buf, 0xcc, sizeof(buf));
+ ATF_REQUIRE_EQ(0, c8rtomb(buf, 0xe2, &s));
+ ATF_REQUIRE_EQ(0, c8rtomb(buf, 0x82, &s));
+ ATF_REQUIRE_EQ(1, c8rtomb(buf, 0xac, &s));
+ ATF_REQUIRE_EQ(0xa4, (unsigned char)buf[0]);
+ ATF_REQUIRE_EQ(0xcc, (unsigned char)buf[1]);
+}
+
+ATF_TC_WITHOUT_HEAD(c8rtomb_utf_8_test);
+ATF_TC_BODY(c8rtomb_utf_8_test, tc)
+{
+ require_lc_ctype("en_US.UTF-8");
+
+ /* Two-byte character U+00E9. */
+ memset(&s, 0, sizeof(s));
+ memset(buf, 0xcc, sizeof(buf));
+ ATF_REQUIRE_EQ(0, c8rtomb(buf, 0xc3, &s));
+ ATF_REQUIRE_EQ(2, c8rtomb(buf, 0xa9, &s));
+ ATF_REQUIRE_EQ(0xc3, (unsigned char)buf[0]);
+ ATF_REQUIRE_EQ(0xa9, (unsigned char)buf[1]);
+ ATF_REQUIRE_EQ(0xcc, (unsigned char)buf[2]);
+
+ /* Four-byte character U+1F4A9. */
+ memset(&s, 0, sizeof(s));
+ memset(buf, 0xcc, sizeof(buf));
+ ATF_REQUIRE_EQ(0, c8rtomb(buf, 0xf0, &s));
+ ATF_REQUIRE_EQ(0, c8rtomb(buf, 0x9f, &s));
+ ATF_REQUIRE_EQ(0, c8rtomb(buf, 0x92, &s));
+ ATF_REQUIRE_EQ(4, c8rtomb(buf, 0xa9, &s));
+ ATF_REQUIRE_EQ(0xf0, (unsigned char)buf[0]);
+ ATF_REQUIRE_EQ(0x9f, (unsigned char)buf[1]);
+ ATF_REQUIRE_EQ(0x92, (unsigned char)buf[2]);
+ ATF_REQUIRE_EQ(0xa9, (unsigned char)buf[3]);
+ ATF_REQUIRE_EQ(0xcc, (unsigned char)buf[4]);
+
+ /* Incomplete sequence followed by ASCII. */
+ memset(&s, 0, sizeof(s));
+ memset(buf, 0xcc, sizeof(buf));
+ ATF_REQUIRE_EQ(0, c8rtomb(buf, 0xc3, &s));
+ ATF_REQUIRE_EQ((size_t)-1, c8rtomb(buf, 'A', &s));
+ ATF_REQUIRE_EQ(EILSEQ, errno);
+ ATF_REQUIRE_EQ(0xcc, (unsigned char)buf[0]);
+
+ /* Incomplete sequence followed by U+0000. */
+ memset(&s, 0, sizeof(s));
+ memset(buf, 0xcc, sizeof(buf));
+ ATF_REQUIRE_EQ(0, c8rtomb(buf, 0xc3, &s));
+ ATF_REQUIRE_EQ((size_t)-1, c8rtomb(buf, 0, &s));
+ ATF_REQUIRE_EQ(EILSEQ, errno);
+ ATF_REQUIRE_EQ(0xcc, (unsigned char)buf[0]);
+
+ /* Overlong encoding of NUL (C0 80). */
+ require_ill_formed(overlong_nul, sizeof(overlong_nul));
+
+ /* Isolated trail surrogate as UTF-8 (ED A0 80 is U+D800). */
+ require_ill_formed(utf8_surrogate, sizeof(utf8_surrogate));
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+ ATF_TP_ADD_TC(tp, c8rtomb_c_locale_test);
+ ATF_TP_ADD_TC(tp, c8rtomb_iso_8859_1_test);
+ ATF_TP_ADD_TC(tp, c8rtomb_iso_8859_15_test);
+ ATF_TP_ADD_TC(tp, c8rtomb_utf_8_test);
+
+ return (atf_no_error());
+}
diff --git a/lib/libc/tests/locale/mbrtoc8_test.c b/lib/libc/tests/locale/mbrtoc8_test.c
new file mode 100644
--- /dev/null
+++ b/lib/libc/tests/locale/mbrtoc8_test.c
@@ -0,0 +1,206 @@
+/*
+ * Copyright (c) 2026 Faraz Vahedi <kfv@FreeBSD.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+/*
+ * Test program for mbrtoc8() as specified by ISO/IEC 9899:2024, §7.30.1.1.
+ */
+
+#include <errno.h>
+#include <limits.h>
+#include <locale.h>
+#include <stdio.h>
+#include <string.h>
+#include <uchar.h>
+
+#include <atf-c.h>
+
+static void
+require_lc_ctype(const char *locale_name)
+{
+ char *lc_ctype_set;
+
+ lc_ctype_set = setlocale(LC_CTYPE, locale_name);
+ if (lc_ctype_set == NULL)
+ atf_tc_fail("setlocale(LC_CTYPE, \"%s\") failed; errno=%d",
+ locale_name, errno);
+
+ ATF_REQUIRE_STREQ(locale_name, lc_ctype_set);
+}
+
+static mbstate_t s;
+static char8_t c8;
+
+ATF_TC_WITHOUT_HEAD(mbrtoc8_c_locale_test);
+ATF_TC_BODY(mbrtoc8_c_locale_test, tc)
+{
+ require_lc_ctype("C");
+
+ /* Null character, internal state. */
+ ATF_REQUIRE_EQ(0, mbrtoc8(&c8, "", 1, NULL));
+ ATF_REQUIRE_EQ(0, c8);
+
+ /* Null character. */
+ memset(&s, 0, sizeof(s));
+ ATF_REQUIRE_EQ(0, mbrtoc8(&c8, "", 1, &s));
+ ATF_REQUIRE_EQ(0, c8);
+
+ /* Latin letter A, internal state. */
+ ATF_REQUIRE_EQ(0, mbrtoc8(NULL, "", 1, NULL));
+ ATF_REQUIRE_EQ(1, mbrtoc8(&c8, "A", 1, NULL));
+ ATF_REQUIRE_EQ('A', c8);
+
+ /* Latin letter A. */
+ memset(&s, 0, sizeof(s));
+ ATF_REQUIRE_EQ(1, mbrtoc8(&c8, "A", 1, &s));
+ ATF_REQUIRE_EQ('A', c8);
+
+ /* Incomplete character sequence. */
+ c8 = 'z';
+ memset(&s, 0, sizeof(s));
+ ATF_REQUIRE_EQ((size_t)-2, mbrtoc8(&c8, "", 0, &s));
+ ATF_REQUIRE_EQ('z', c8);
+
+ /* Check that mbrtoc8() doesn't read ahead aggressively. */
+ memset(&s, 0, sizeof(s));
+ ATF_REQUIRE_EQ(1, mbrtoc8(&c8, "AB", 2, &s));
+ ATF_REQUIRE_EQ('A', c8);
+ ATF_REQUIRE_EQ(1, mbrtoc8(&c8, "C", 1, &s));
+ ATF_REQUIRE_EQ('C', c8);
+}
+
+ATF_TC_WITHOUT_HEAD(mbrtoc8_iso_8859_1_test);
+ATF_TC_BODY(mbrtoc8_iso_8859_1_test, tc)
+{
+ require_lc_ctype("en_US.ISO8859-1");
+
+ /* Currency sign U+00A4 -> UTF-8 C2 A4. */
+ memset(&s, 0, sizeof(s));
+ ATF_REQUIRE_EQ(1, mbrtoc8(&c8, "\xa4", 1, &s));
+ ATF_REQUIRE_EQ(0xc2, c8);
+ ATF_REQUIRE_EQ((size_t)-3, mbrtoc8(&c8, "", 0, &s));
+ ATF_REQUIRE_EQ(0xa4, c8);
+}
+
+ATF_TC_WITHOUT_HEAD(mbrtoc8_iso_8859_15_test);
+ATF_TC_BODY(mbrtoc8_iso_8859_15_test, tc)
+{
+ require_lc_ctype("en_US.ISO8859-15");
+
+ /* Euro sign U+20AC -> UTF-8 E2 82 AC. */
+ memset(&s, 0, sizeof(s));
+ ATF_REQUIRE_EQ(1, mbrtoc8(&c8, "\xa4", 1, &s));
+ ATF_REQUIRE_EQ(0xe2, c8);
+ ATF_REQUIRE_EQ((size_t)-3, mbrtoc8(&c8, "", 0, &s));
+ ATF_REQUIRE_EQ(0x82, c8);
+ ATF_REQUIRE_EQ((size_t)-3, mbrtoc8(&c8, "", 0, &s));
+ ATF_REQUIRE_EQ(0xac, c8);
+}
+
+ATF_TC_WITHOUT_HEAD(mbrtoc8_utf_8_test);
+ATF_TC_BODY(mbrtoc8_utf_8_test, tc)
+{
+ require_lc_ctype("en_US.UTF-8");
+
+ /* Null character, internal state. */
+ ATF_REQUIRE_EQ(0, mbrtoc8(NULL, "", 1, NULL));
+ ATF_REQUIRE_EQ(0, mbrtoc8(&c8, "", 1, NULL));
+ ATF_REQUIRE_EQ(0, c8);
+
+ /* Null character. */
+ memset(&s, 0, sizeof(s));
+ ATF_REQUIRE_EQ(0, mbrtoc8(&c8, "", 1, &s));
+ ATF_REQUIRE_EQ(0, c8);
+
+ /* Latin letter A. */
+ memset(&s, 0, sizeof(s));
+ ATF_REQUIRE_EQ(1, mbrtoc8(&c8, "A", 1, &s));
+ ATF_REQUIRE_EQ('A', c8);
+
+ /* Incomplete character sequence (zero length). */
+ c8 = 'z';
+ memset(&s, 0, sizeof(s));
+ ATF_REQUIRE_EQ((size_t)-2, mbrtoc8(&c8, "", 0, &s));
+ ATF_REQUIRE_EQ('z', c8);
+
+ /* Incomplete character sequence (truncated double-byte). */
+ memset(&s, 0, sizeof(s));
+ c8 = 0;
+ ATF_REQUIRE_EQ((size_t)-2, mbrtoc8(&c8, "\xc3", 1, &s));
+
+ /* Same as above, but complete: U+00C4 -> C3 84. */
+ memset(&s, 0, sizeof(s));
+ c8 = 0;
+ ATF_REQUIRE_EQ(2, mbrtoc8(&c8, "\xc3\x84", 2, &s));
+ ATF_REQUIRE_EQ(0xc3, c8);
+ ATF_REQUIRE_EQ((size_t)-3, mbrtoc8(&c8, "", 0, &s));
+ ATF_REQUIRE_EQ(0x84, c8);
+
+ /* Test restarting behaviour. */
+ memset(&s, 0, sizeof(s));
+ c8 = 0;
+ ATF_REQUIRE_EQ((size_t)-2, mbrtoc8(&c8, "\xc3", 1, &s));
+ ATF_REQUIRE_EQ(0, c8);
+ ATF_REQUIRE_EQ(1, mbrtoc8(&c8, "\xb7", 1, &s));
+ ATF_REQUIRE_EQ(0xc3, c8);
+ ATF_REQUIRE_EQ((size_t)-3, mbrtoc8(&c8, "", 0, &s));
+ ATF_REQUIRE_EQ(0xb7, c8);
+
+ /* Four-byte sequence: U+1F4A9. */
+ memset(&s, 0, sizeof(s));
+ c8 = 0;
+ ATF_REQUIRE_EQ(4, mbrtoc8(&c8, "\xf0\x9f\x92\xa9", 4, &s));
+ ATF_REQUIRE_EQ(0xf0, c8);
+ ATF_REQUIRE_EQ((size_t)-3, mbrtoc8(&c8, "", 0, &s));
+ ATF_REQUIRE_EQ(0x9f, c8);
+ ATF_REQUIRE_EQ((size_t)-3, mbrtoc8(&c8, "", 0, &s));
+ ATF_REQUIRE_EQ(0x92, c8);
+ ATF_REQUIRE_EQ((size_t)-3, mbrtoc8(&c8, "", 0, &s));
+ ATF_REQUIRE_EQ(0xa9, c8);
+
+ /* Letter e with acute, precomposed. */
+ memset(&s, 0, sizeof(s));
+ c8 = 0;
+ ATF_REQUIRE_EQ(2, mbrtoc8(&c8, "\xc3\xa9", 2, &s));
+ ATF_REQUIRE_EQ(0xc3, c8);
+ ATF_REQUIRE_EQ((size_t)-3, mbrtoc8(&c8, "", 0, &s));
+ ATF_REQUIRE_EQ(0xa9, c8);
+
+ /* Letter e with acute, combined. */
+ memset(&s, 0, sizeof(s));
+ c8 = 0;
+ ATF_REQUIRE_EQ(1, mbrtoc8(&c8, "\x65\xcc\x81", 3, &s));
+ ATF_REQUIRE_EQ(0x65, c8);
+ ATF_REQUIRE_EQ(2, mbrtoc8(&c8, "\xcc\x81", 2, &s));
+ ATF_REQUIRE_EQ(0xcc, c8);
+ ATF_REQUIRE_EQ((size_t)-3, mbrtoc8(&c8, "", 0, &s));
+ ATF_REQUIRE_EQ(0x81, c8);
+
+ /* pc8 == NULL still records remaining UTF-8 units. */
+ memset(&s, 0, sizeof(s));
+ ATF_REQUIRE_EQ(2, mbrtoc8(NULL, "\xc3\x84", 2, &s));
+ ATF_REQUIRE_EQ((size_t)-3, mbrtoc8(&c8, "", 0, &s));
+ ATF_REQUIRE_EQ(0x84, c8);
+
+ /*
+ * Null s reinitializes conversion state, as mbrtoc16() does.
+ * Pending units are discarded and pc8 is ignored.
+ */
+ memset(&s, 0, sizeof(s));
+ ATF_REQUIRE_EQ(2, mbrtoc8(&c8, "\xc3\x84", 2, &s));
+ c8 = 0xff;
+ ATF_REQUIRE_EQ(0, mbrtoc8(&c8, NULL, 99, &s));
+ ATF_REQUIRE_EQ(0xff, c8);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+ ATF_TP_ADD_TC(tp, mbrtoc8_c_locale_test);
+ ATF_TP_ADD_TC(tp, mbrtoc8_iso_8859_1_test);
+ ATF_TP_ADD_TC(tp, mbrtoc8_iso_8859_15_test);
+ ATF_TP_ADD_TC(tp, mbrtoc8_utf_8_test);
+
+ return (atf_no_error());
+}
diff --git a/sys/sys/_types.h b/sys/sys/_types.h
--- a/sys/sys/_types.h
+++ b/sys/sys/_types.h
@@ -209,9 +209,8 @@
#define _CHAR16_T_DECLARED
#define _CHAR32_T_DECLARED
#endif
-/* and so is char8_t in C++20 */
-#if defined(__cplusplus) && __cplusplus >= 202002L
-#define _CHAR8_T_DECLARED
+#if defined(__cpp_char8_t)
+#define _CHAR8_T_DECLARED
#endif
typedef struct {

File Metadata

Mime Type
text/plain
Expires
Fri, Sep 25, 5:59 AM (10 h, 50 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
39583431
Default Alt Text
D59213.diff (27 KB)

Event Timeline