Unicode encoding utilities. More...

Macros
#define	SOURCEMETA_CORE_JOINING_TYPE_LIST(X)
#define	SOURCEMETA_CORE_BIDI_CLASS_LIST(X)
#define	SOURCEMETA_CORE_UNICODE_SCRIPT_LIST(X)
#define	SOURCEMETA_CORE_NFC_QUICK_CHECK_LIST(X)

Enumerations
enum class	sourcemeta::core::JoiningType : std::uint8_t
enum class	sourcemeta::core::BidiClass : std::uint8_t
enum class	sourcemeta::core::UnicodeScript : std::uint8_t
enum class	sourcemeta::core::NFCQuickCheck : std::uint8_t

Functions
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::codepoint_to_utf8 (const char32_t codepoint) -> std::string
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::codepoint_to_utf8 (const char32_t codepoint, std::ostream &output) -> void
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::codepoint_to_utf8 (const char32_t codepoint, std::string &output) -> void
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::utf8_to_utf32 (std::istream &input) -> std::optional< std::u32string >
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::utf8_to_utf32 (const std::string_view input) -> std::optional< std::u32string >
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::utf32_to_utf8 (const std::u32string_view input) -> std::string
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::utf32_to_utf8_lenient (const std::u32string_view input) -> std::string
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::to_valid_utf8 (const std::string_view input) -> std::string
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::utf8_to_wide (const std::string_view input) -> std::wstring
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::wide_to_utf8 (const std::wstring_view input) -> std::string
constexpr auto	sourcemeta::core::utf8_lead_byte_size (const unsigned char byte) -> std::uint8_t
constexpr auto	sourcemeta::core::is_utf8_tail (const unsigned char lead, const std::size_t position, const unsigned char byte) -> bool
constexpr auto	sourcemeta::core::utf8_sequence_size (const std::string_view input) -> std::size_t
constexpr auto	sourcemeta::core::is_utf8_continuation (const unsigned char byte) -> bool
constexpr auto	sourcemeta::core::utf8_codepoint_count (const std::string_view input) -> std::size_t
constexpr auto	sourcemeta::core::utf8_codepoint_within (const std::string_view input, const std::size_t minimum, const std::size_t maximum) -> bool
constexpr auto	sourcemeta::core::is_surrogate (const char32_t codepoint) -> bool
constexpr auto	sourcemeta::core::is_valid_codepoint (const char32_t codepoint) -> bool
constexpr auto	sourcemeta::core::is_ucschar (const char32_t codepoint) -> bool
constexpr auto	sourcemeta::core::is_iprivate (const char32_t codepoint) -> bool
constexpr auto	sourcemeta::core::utf8_codepoint_byte_count (const char32_t codepoint) -> std::uint8_t
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::combining_class (const char32_t codepoint) noexcept -> std::uint8_t
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::joining_type (const char32_t codepoint) noexcept -> JoiningType
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::bidi_class (const char32_t codepoint) noexcept -> BidiClass
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::script (const char32_t codepoint) noexcept -> UnicodeScript
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::is_combining_mark (const char32_t codepoint) noexcept -> bool
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::nfc_quick_check (const char32_t codepoint) noexcept -> NFCQuickCheck
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::canonical_decomposition (const char32_t codepoint) noexcept -> std::u32string_view
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::canonical_composition (const char32_t starter, const char32_t combining) noexcept -> std::optional< char32_t >
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::nfc (const std::u32string_view input) -> std::u32string
SOURCEMETA_CORE_UNICODE_EXPORT auto	sourcemeta::core::is_nfc (const std::u32string_view input) -> bool
constexpr auto	sourcemeta::core::utf8_codepoint_length (const std::string_view input, const std::string_view::size_type position) -> std::size_t
constexpr auto	sourcemeta::core::utf8_decode (const std::string_view input, const std::string_view::size_type position) -> std::optional< std::pair< char32_t, std::size_t > >

Detailed Description

Unicode encoding utilities.

This functionality is included as follows:

#include <sourcemeta/core/unicode.h>

Macro Definition Documentation

◆ SOURCEMETA_CORE_BIDI_CLASS_LIST

#define SOURCEMETA_CORE_BIDI_CLASS_LIST ( X )

Value:

X(LeftToRight, "L")                                                          \
X(RightToLeft, "R")                                                          \
X(ArabicLetter, "AL")                                                        \
X(EuropeanNumber, "EN")                                                      \
X(EuropeanSeparator, "ES")                                                   \
X(EuropeanTerminator, "ET")                                                  \
X(ArabicNumber, "AN")                                                        \
X(CommonSeparator, "CS")                                                     \
X(NonspacingMark, "NSM")                                                     \
X(BoundaryNeutral, "BN")                                                     \
X(ParagraphSeparator, "B")                                                   \
X(SegmentSeparator, "S")                                                     \
X(WhiteSpace, "WS")                                                          \
X(OtherNeutral, "ON")                                                        \
X(LeftToRightEmbedding, "LRE")                                               \
X(LeftToRightOverride, "LRO")                                                \
X(RightToLeftEmbedding, "RLE")                                               \
X(RightToLeftOverride, "RLO")                                                \
X(PopDirectionalFormat, "PDF")                                               \
X(LeftToRightIsolate, "LRI")                                                 \
X(RightToLeftIsolate, "RLI")                                                 \
X(FirstStrongIsolate, "FSI")                                                 \
X(PopDirectionalIsolate, "PDI")

Each entry maps a BidiClass enum name to its UCD short alias.

◆ SOURCEMETA_CORE_JOINING_TYPE_LIST

#define SOURCEMETA_CORE_JOINING_TYPE_LIST ( X )

Value:

X(NonJoining, "U")                                                           \
X(Transparent, "T")                                                          \
X(LeftJoining, "L")                                                          \
X(RightJoining, "R")                                                         \
X(DualJoining, "D")                                                          \
X(JoinCausing, "C")

Each entry maps a JoiningType enum name to its UCD short alias.

◆ SOURCEMETA_CORE_NFC_QUICK_CHECK_LIST

#define SOURCEMETA_CORE_NFC_QUICK_CHECK_LIST ( X )

Value:

X(Yes, "Y")                                                                  \
X(No, "N")                                                                   \
X(Maybe, "M")

Each entry maps an NFCQuickCheck enum name to its UCD short alias.

◆ SOURCEMETA_CORE_UNICODE_SCRIPT_LIST

#define SOURCEMETA_CORE_UNICODE_SCRIPT_LIST ( X )

Each entry maps a UnicodeScript enum name to its UCD long alias. Per UAX #24 §1.4, Katakana_Or_Hiragana only appears in the Script_Extensions property and never in the Script property itself.

Enumeration Type Documentation

◆ BidiClass

enum class sourcemeta::core::BidiClass : std::uint8_t

strong

The bidirectional class of a Unicode codepoint per UAX #44. See https://www.unicode.org/reports/tr44/ for the property's definition.

◆ JoiningType

enum class sourcemeta::core::JoiningType : std::uint8_t

strong

The joining type of a Unicode codepoint per UAX #44. See https://www.unicode.org/reports/tr44/ for the property's definition.

◆ NFCQuickCheck

enum class sourcemeta::core::NFCQuickCheck : std::uint8_t

strong

The NFC quick-check result for a Unicode codepoint per UAX #15. See https://www.unicode.org/reports/tr15/ for the property's definition.

◆ UnicodeScript

enum class sourcemeta::core::UnicodeScript : std::uint8_t

strong

The script of a Unicode codepoint per UAX #24. See https://www.unicode.org/reports/tr24/ for the property's definition.

Function Documentation

◆ bidi_class()

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::bidi_class ( const char32_t codepoint ) -> BidiClass

noexcept

Return the bidirectional class of a Unicode codepoint. See https://www.unicode.org/reports/tr44/ for the property's definition. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::bidi_class(U'A') ==
       sourcemeta::core::BidiClass::LeftToRight);
assert(sourcemeta::core::bidi_class(U'\u05D0') ==
       sourcemeta::core::BidiClass::RightToLeft);
assert(sourcemeta::core::bidi_class(U'\u0627') ==
       sourcemeta::core::BidiClass::ArabicLetter);

◆ canonical_composition()

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::canonical_composition	(	const char32_t	starter,
		const char32_t	combining ) -> std::optional< char32_t >

noexcept

Return the primary composite of a starter codepoint and a following codepoint per UAX #15, or std::nullopt if the pair does not form a primary composite. Hangul jamo composition is algorithmic per UAX #15 §3.12 and is not included in this table. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::canonical_composition(U'A', U'\u0300') ==
       U'\u00C0');
assert(sourcemeta::core::canonical_composition(U'u', U'\u0308') ==
       U'\u00FC');
assert(sourcemeta::core::canonical_composition(U'A', U'B') ==
       std::nullopt);

◆ canonical_decomposition()

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::canonical_decomposition ( const char32_t codepoint ) -> std::u32string_view

noexcept

Return the non-recursive canonical decomposition of a Unicode codepoint per UAX #15. The view points into static data and remains valid for the program's lifetime. An empty view means the codepoint has no canonical decomposition. Hangul precomposed syllables decompose algorithmically per UAX #15 and are reported as empty here. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::canonical_decomposition(U'A').empty());
assert(sourcemeta::core::canonical_decomposition(U'\u00FC') ==
       std::u32string_view{U"u\u0308"});
assert(sourcemeta::core::canonical_decomposition(U'\u2126') ==
       std::u32string_view{U"\u03A9"});

◆ codepoint_to_utf8() [1/3]

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::codepoint_to_utf8 ( const char32_t codepoint ) -> std::string

Encode a single Unicode codepoint as a UTF-8 string. The codepoint must be a valid Unicode scalar value, otherwise the output is unspecified. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::codepoint_to_utf8(0x41) == "A");

◆ codepoint_to_utf8() [2/3]

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::codepoint_to_utf8	(	const char32_t	codepoint,
		std::ostream &	output ) -> void

Encode a single Unicode codepoint as UTF-8 into an output stream. The codepoint must be a valid Unicode scalar value, otherwise the output is unspecified. For example:

#include <sourcemeta/core/unicode.h>
#include <sstream>
#include <cassert>
 
std::ostringstream output;
sourcemeta::core::codepoint_to_utf8(0x41, output);
assert(output.str() == "A");

◆ codepoint_to_utf8() [3/3]

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::codepoint_to_utf8	(	const char32_t	codepoint,
		std::string &	output ) -> void

Encode a single Unicode codepoint as UTF-8, appending to an existing string. The codepoint must be a valid Unicode scalar value, otherwise the output is unspecified. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
std::string output;
sourcemeta::core::codepoint_to_utf8(0x41, output);
assert(output == "A");

◆ combining_class()

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::combining_class ( const char32_t codepoint ) -> std::uint8_t

noexcept

Return the canonical combining class of a Unicode codepoint. See https://www.unicode.org/reports/tr44/ for the property's definition. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::combining_class(U'\u094D') == 9);
assert(sourcemeta::core::combining_class(U'\u0301') == 230);
assert(sourcemeta::core::combining_class(U'A') == 0);

◆ is_combining_mark()

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::is_combining_mark ( const char32_t codepoint ) -> bool

noexcept

Return whether a Unicode codepoint is a combining mark, in the sense of UAX #44 general category Mn (Nonspacing_Mark), Mc (Spacing_Mark), or Me (Enclosing_Mark). See https://www.unicode.org/reports/tr44/ for the property's definition. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::is_combining_mark(U'\u0301'));
assert(sourcemeta::core::is_combining_mark(U'\u094D'));
assert(!sourcemeta::core::is_combining_mark(U'A'));

◆ is_iprivate()

auto sourcemeta::core::is_iprivate ( const char32_t codepoint ) -> bool

inlineconstexpr

Check whether the given codepoint matches the iprivate production of RFC 3987 Section 2.2, the set of private-use characters that an IRI may carry in the query component. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::is_iprivate(0xE000));
assert(sourcemeta::core::is_iprivate(0xF0000));
assert(!sourcemeta::core::is_iprivate(0x0041));
assert(!sourcemeta::core::is_iprivate(0xF8FF + 1));

◆ is_nfc()

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::is_nfc ( const std::u32string_view input ) -> bool

Return whether input is already in Unicode Normalisation Form C per UAX #15. Uses a quick-check fast path and falls back to a full normalise-and-compare when the quick check is inconclusive. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::is_nfc(U"À"));
assert(!sourcemeta::core::is_nfc(U"À"));
assert(sourcemeta::core::is_nfc(U"가"));

◆ is_surrogate()

auto sourcemeta::core::is_surrogate ( const char32_t codepoint ) -> bool

inlineconstexpr

Check whether the given codepoint is in the UTF-16 surrogate range (U+D800 to U+DFFF), which is forbidden in scalar Unicode text. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::is_surrogate(0xD800));
assert(sourcemeta::core::is_surrogate(0xDFFF));
assert(!sourcemeta::core::is_surrogate(0xD7FF));
assert(!sourcemeta::core::is_surrogate(0xE000));

◆ is_ucschar()

auto sourcemeta::core::is_ucschar ( const char32_t codepoint ) -> bool

inlineconstexpr

Check whether the given codepoint matches the ucschar production of RFC 3987 Section 2.2, the set of non-ASCII characters that an IRI may carry in components other than the scheme, host, and percent-encoded octets. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::is_ucschar(0x00A0));
assert(sourcemeta::core::is_ucschar(0x4E2D));
assert(!sourcemeta::core::is_ucschar(0x0041));
assert(!sourcemeta::core::is_ucschar(0xE000));

◆ is_utf8_continuation()

auto sourcemeta::core::is_utf8_continuation ( const unsigned char byte ) -> bool

inlineconstexpr

Check whether the given byte is a UTF-8 continuation byte (x80-BF per RFC 3629 Section 4). For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::is_utf8_continuation(0x80));
assert(sourcemeta::core::is_utf8_continuation(0xBF));
assert(!sourcemeta::core::is_utf8_continuation(0x7F));
assert(!sourcemeta::core::is_utf8_continuation(0xC0));

◆ is_utf8_tail()

auto sourcemeta::core::is_utf8_tail	(	const unsigned char	lead,
		const std::size_t	position,
		const unsigned char	byte ) -> bool

inlineconstexpr

Check whether the given byte may follow the given lead byte at the given position of a UTF-8 sequence, where position 1 is the byte right after the lead. RFC 3629 Section 4 narrows that first position below the general x80-BF for four of the leads, so as to exclude the overlong encodings and the surrogate range that the shorter grammar would otherwise admit:

UTF8-3 = xE0 xA0-BF UTF8-tail / xE1-EC 2( UTF8-tail ) / xED x80-9F UTF8-tail / xEE-EF 2( UTF8-tail ) UTF8-4 = xF0 x90-BF 2( UTF8-tail ) / xF1-F3 3( UTF8-tail ) / xF4 x80-8F 2( UTF8-tail )

For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::is_utf8_tail(0xE2, 1, 0x82));
// A surrogate lead admits no byte above %x9F
assert(!sourcemeta::core::is_utf8_tail(0xED, 1, 0xA0));

◆ is_valid_codepoint()

auto sourcemeta::core::is_valid_codepoint ( const char32_t codepoint ) -> bool

inlineconstexpr

Check whether the given value is a valid Unicode codepoint: in the range U+0000 to U+10FFFF, excluding the UTF-16 surrogate range (U+D800 to U+DFFF). For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::is_valid_codepoint(0x0000));
assert(sourcemeta::core::is_valid_codepoint(0x10FFFF));
assert(!sourcemeta::core::is_valid_codepoint(0xD800));
assert(!sourcemeta::core::is_valid_codepoint(0x110000));

◆ joining_type()

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::joining_type ( const char32_t codepoint ) -> JoiningType

noexcept

Return the joining type of a Unicode codepoint. See https://www.unicode.org/reports/tr44/ for the property's definition. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::joining_type(U'\u0628') ==
       sourcemeta::core::JoiningType::DualJoining);
assert(sourcemeta::core::joining_type(U'\u200D') ==
       sourcemeta::core::JoiningType::JoinCausing);
assert(sourcemeta::core::joining_type(U'A') ==
       sourcemeta::core::JoiningType::NonJoining);

◆ nfc()

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::nfc ( const std::u32string_view input ) -> std::u32string

Return the Unicode Normalisation Form C of input per UAX #15. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::nfc(U"A\u0300") == U"\u00C0");
assert(sourcemeta::core::nfc(U"\u00C0") == U"\u00C0");
assert(sourcemeta::core::nfc(U"\u1100\u1161") == U"\uAC00");

◆ nfc_quick_check()

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::nfc_quick_check ( const char32_t codepoint ) -> NFCQuickCheck

noexcept

Return the NFC quick-check property of a Unicode codepoint per UAX #15. See https://www.unicode.org/reports/tr15/ for the property's definition. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::nfc_quick_check(U'A') ==
       sourcemeta::core::NFCQuickCheck::Yes);
assert(sourcemeta::core::nfc_quick_check(U'\u2126') ==
       sourcemeta::core::NFCQuickCheck::No);
assert(sourcemeta::core::nfc_quick_check(U'\u0300') ==
       sourcemeta::core::NFCQuickCheck::Maybe);

◆ script()

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::script ( const char32_t codepoint ) -> UnicodeScript

noexcept

Return the script of a Unicode codepoint. See https://www.unicode.org/reports/tr24/ for the property's definition. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::script(U'A') ==
       sourcemeta::core::UnicodeScript::Latin);
assert(sourcemeta::core::script(U'\u0391') ==
       sourcemeta::core::UnicodeScript::Greek);
assert(sourcemeta::core::script(U'\u05D0') ==
       sourcemeta::core::UnicodeScript::Hebrew);

◆ to_valid_utf8()

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::to_valid_utf8 ( const std::string_view input ) -> std::string

Read a byte sequence as UTF-8, standing in the replacement character U+FFFD for every sequence that is not well-formed. One replacement stands in for a whole maximal subpart, so a sequence cut short yields a single one rather than one per stray byte. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
// The escapes are broken across literals, as a hexadecimal one otherwise
// swallows the letter that follows it
assert(sourcemeta::core::to_valid_utf8("a\xFF" "b") ==
       "a\xEF\xBF\xBD" "b");

◆ utf32_to_utf8()

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::utf32_to_utf8 ( const std::u32string_view input ) -> std::string

Encode a sequence of Unicode codepoints (UTF-32) as a UTF-8 string, the inverse of utf8_to_utf32. Every codepoint must be a valid Unicode scalar value (in particular, not a surrogate), otherwise the output is unspecified. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::utf32_to_utf8(U"A\u00E9") == "A\xC3\xA9");

◆ utf32_to_utf8_lenient()

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::utf32_to_utf8_lenient ( const std::u32string_view input ) -> std::string

Encode a sequence of codepoints as UTF-8, the lenient counterpart of utf32_to_utf8. Unlike that function, surrogate codepoints are permitted and encoded as their three-byte WTF-8 sequence rather than being rejected. Each codepoint must still be within the Unicode codespace (U+0000 to U+10FFFF), otherwise the output is unspecified. Because a surrogate is not a valid scalar value, its encoding is ill-formed UTF-8: it does not round-trip through utf8_to_utf32 (which rejects surrogates) and is meant for byte-preserving workflows, such as feeding a strict decoder or validator with input that carries lone surrogates. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
// A lone surrogate U+D800 becomes its ill-formed three-byte encoding
assert(sourcemeta::core::utf32_to_utf8_lenient(std::u32string{0xD800}) ==
       "\xED\xA0\x80");

◆ utf8_codepoint_byte_count()

auto sourcemeta::core::utf8_codepoint_byte_count ( const char32_t codepoint ) -> std::uint8_t

inlineconstexpr

Determine the number of UTF-8 bytes that a codepoint encodes to per RFC 3629: 1 byte for U+0000-U+007F, 2 bytes for U+0080-U+07FF, 3 bytes for U+0800-U+FFFF, and 4 bytes for U+10000 and above. The caller is responsible for ensuring the codepoint is in range. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::utf8_codepoint_byte_count(0x0041) == 1);
assert(sourcemeta::core::utf8_codepoint_byte_count(0x00E9) == 2);
assert(sourcemeta::core::utf8_codepoint_byte_count(0x4E2D) == 3);
assert(sourcemeta::core::utf8_codepoint_byte_count(0x1F600) == 4);

◆ utf8_codepoint_count()

auto sourcemeta::core::utf8_codepoint_count ( const std::string_view input ) -> std::size_t

inlineconstexpr

Count the number of Unicode code points in a UTF-8 string, assuming the input is well-formed. Each code point begins at a byte that is not a continuation byte (x80-BF per RFC 3629 Section 4). For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::utf8_codepoint_count("abc") == 3);
assert(sourcemeta::core::utf8_codepoint_count("caf\xc3\xa9") == 4);

◆ utf8_codepoint_length()

auto sourcemeta::core::utf8_codepoint_length	(	const std::string_view	input,
		const std::string_view::size_type	position ) -> std::size_t

inlineconstexpr

Determine the byte length of the valid UTF-8 codepoint starting at the given position within the input. Returns 1 for an ASCII byte, 2/3/4 for a valid multi-byte UTF-8 sequence (RFC 3629 Section 4, excluding overlong encodings, surrogates, and code points above U+10FFFF), or 0 if the bytes at that position do not start a valid UTF-8 codepoint. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::utf8_codepoint_length("A", 0) == 1);
assert(sourcemeta::core::utf8_codepoint_length("\xce\xb1", 0) == 2);
assert(sourcemeta::core::utf8_codepoint_length("\xe4\xb8\xad", 0) == 3);
assert(sourcemeta::core::utf8_codepoint_length("\xf0\x9f\x98\x80", 0) == 4);
assert(sourcemeta::core::utf8_codepoint_length("\xed\xa0\x80", 0) == 0);

◆ utf8_codepoint_within()

auto sourcemeta::core::utf8_codepoint_within	(	const std::string_view	input,
		const std::size_t	minimum,
		const std::size_t	maximum ) -> bool

inlineconstexpr

Check whether the number of Unicode code points in a well-formed UTF-8 string is within an inclusive range, without necessarily scanning the whole string. Because a code point occupies between one and four bytes, the byte length bounds the code point count, so the extreme cases are decided in constant time and the fallback scan stops as soon as the range is exceeded. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::utf8_codepoint_within("abc", 1, 5));
assert(!sourcemeta::core::utf8_codepoint_within("abc", 4, 5));

◆ utf8_decode()

auto sourcemeta::core::utf8_decode	(	const std::string_view	input,
		const std::string_view::size_type	position ) -> std::optional< std::pair< char32_t, std::size_t > >

inlineconstexpr

Decode the single UTF-8 codepoint that begins at the given position within the input, returning the codepoint together with the number of bytes it occupies, or an empty result when the bytes at that position do not start a valid UTF-8 codepoint (RFC 3629 Section 4, excluding overlong encodings, surrogates, and code points above U+10FFFF). For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
const auto result{sourcemeta::core::utf8_decode("\xCE\xB1", 0)};
assert(result.has_value());
assert(result.value().first == 0x03B1);
assert(result.value().second == 2);
assert(!sourcemeta::core::utf8_decode("\xED\xA0\x80", 0).has_value());

◆ utf8_lead_byte_size()

auto sourcemeta::core::utf8_lead_byte_size ( const unsigned char byte ) -> std::uint8_t

inlineconstexpr

Determine the byte length encoded by a UTF-8 lead byte. Returns 1 for an ASCII byte (x00-7F), 2 for a 2-byte lead (xC2-DF), 3 for a 3-byte lead (xE0-EF), 4 for a 4-byte lead (xF0-F4), or 0 for any other byte (continuation byte, overlong xC0/xC1, or out-of-range xF5-FF). For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::utf8_lead_byte_size(0x41) == 1);
assert(sourcemeta::core::utf8_lead_byte_size(0xCE) == 2);
assert(sourcemeta::core::utf8_lead_byte_size(0xE4) == 3);
assert(sourcemeta::core::utf8_lead_byte_size(0xF0) == 4);
assert(sourcemeta::core::utf8_lead_byte_size(0x80) == 0);

◆ utf8_sequence_size()

auto sourcemeta::core::utf8_sequence_size ( const std::string_view input ) -> std::size_t

inlineconstexpr

Determine the byte length of the well-formed UTF-8 sequence that the given input begins with, or 0 when it does not begin with one. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::utf8_sequence_size("A") == 1);
assert(sourcemeta::core::utf8_sequence_size("\xC3\xA9") == 2);
// A surrogate is not a scalar value, so it is not well-formed
assert(sourcemeta::core::utf8_sequence_size("\xED\xA0\x80") == 0);

◆ utf8_to_utf32() [1/2]

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::utf8_to_utf32 ( const std::string_view input ) -> std::optional< std::u32string >

Decode a UTF-8 string into a sequence of Unicode codepoints (UTF-32). Returns std::nullopt if the input contains invalid UTF-8. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
const auto result{sourcemeta::core::utf8_to_utf32("A")};
assert(result.has_value());
assert(result.value() == std::u32string{0x41});

◆ utf8_to_utf32() [2/2]

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::utf8_to_utf32 ( std::istream & input ) -> std::optional< std::u32string >

Decode a UTF-8 byte stream into a sequence of Unicode codepoints (UTF-32). Returns std::nullopt if the input contains invalid UTF-8. For example:

#include <sourcemeta/core/unicode.h>
#include <sstream>
#include <cassert>
 
std::istringstream input{"A"};
const auto result{sourcemeta::core::utf8_to_utf32(input)};
assert(result.has_value());
assert(result.value() == std::u32string{0x41});

◆ utf8_to_wide()

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::utf8_to_wide ( const std::string_view input ) -> std::wstring

Convert a UTF-8 string into its wide character form without validation. The input must be valid UTF-8, otherwise the result is undefined. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::utf8_to_wide("hello") == L"hello");

◆ wide_to_utf8()

SOURCEMETA_CORE_UNICODE_EXPORT auto sourcemeta::core::wide_to_utf8 ( const std::wstring_view input ) -> std::string

Convert a wide string into its UTF-8 form without validation. The input must be valid, otherwise the result is undefined. For example:

#include <sourcemeta/core/unicode.h>
#include <cassert>
 
assert(sourcemeta::core::wide_to_utf8(L"hello") == "hello");

Macros

Enumerations

Functions

Detailed Description

Macro Definition Documentation

◆ SOURCEMETA_CORE_BIDI_CLASS_LIST

◆ SOURCEMETA_CORE_JOINING_TYPE_LIST

◆ SOURCEMETA_CORE_NFC_QUICK_CHECK_LIST

◆ SOURCEMETA_CORE_UNICODE_SCRIPT_LIST

Enumeration Type Documentation

◆ BidiClass

◆ JoiningType

◆ NFCQuickCheck

◆ UnicodeScript

Function Documentation

◆ bidi_class()

◆ canonical_composition()

◆ canonical_decomposition()

◆ codepoint_to_utf8() [1/3]

◆ codepoint_to_utf8() [2/3]

◆ codepoint_to_utf8() [3/3]

◆ combining_class()

◆ is_combining_mark()

◆ is_iprivate()

◆ is_nfc()

◆ is_surrogate()

◆ is_ucschar()

◆ is_utf8_continuation()

◆ is_utf8_tail()

◆ is_valid_codepoint()

◆ joining_type()

◆ nfc()

◆ nfc_quick_check()

◆ script()

◆ to_valid_utf8()

◆ utf32_to_utf8()

◆ utf32_to_utf8_lenient()

◆ utf8_codepoint_byte_count()

◆ utf8_codepoint_count()

◆ utf8_codepoint_length()

◆ utf8_codepoint_within()

◆ utf8_decode()

◆ utf8_lead_byte_size()

◆ utf8_sequence_size()

◆ utf8_to_utf32() [1/2]

◆ utf8_to_utf32() [2/2]

◆ utf8_to_wide()

◆ wide_to_utf8()