unicode.h

#ifndef SOURCEMETA_CORE_UNICODE_H_
#define SOURCEMETA_CORE_UNICODE_H_
 
#ifndef SOURCEMETA_CORE_UNICODE_EXPORT
#include <sourcemeta/core/unicode_export.h>
#endif
 
#include <sourcemeta/core/unicode_ucd.h>
 
#include <cstddef>     // std::size_t
#include <cstdint>     // std::uint8_t, std::uint64_t
#include <cstring>     // std::memcpy
#include <istream>     // std::istream
#include <optional>    // std::optional
#include <ostream>     // std::ostream
#include <string>      // std::string, std::u32string, std::wstring
#include <string_view> // std::string_view, std::u32string_view, std::wstring_view
#include <utility>     // std::pair, std::make_pair

 
namespace sourcemeta::core {

SOURCEMETA_CORE_UNICODE_EXPORT
auto codepoint_to_utf8(const char32_t codepoint) -> std::string;

SOURCEMETA_CORE_UNICODE_EXPORT
auto codepoint_to_utf8(const char32_t codepoint, std::ostream &output) -> void;

SOURCEMETA_CORE_UNICODE_EXPORT
auto codepoint_to_utf8(const char32_t codepoint, std::string &output) -> void;

SOURCEMETA_CORE_UNICODE_EXPORT
auto utf8_to_utf32(std::istream &input) -> std::optional<std::u32string>;

SOURCEMETA_CORE_UNICODE_EXPORT
auto utf8_to_utf32(const std::string_view input)
    -> std::optional<std::u32string>;

SOURCEMETA_CORE_UNICODE_EXPORT
auto utf32_to_utf8(const std::u32string_view input) -> std::string;

SOURCEMETA_CORE_UNICODE_EXPORT
auto utf32_to_utf8_lenient(const std::u32string_view input) -> std::string;

SOURCEMETA_CORE_UNICODE_EXPORT
auto to_valid_utf8(const std::string_view input) -> std::string;

SOURCEMETA_CORE_UNICODE_EXPORT
auto utf8_to_wide(const std::string_view input) -> std::wstring;

SOURCEMETA_CORE_UNICODE_EXPORT
auto wide_to_utf8(const std::wstring_view input) -> std::string;

constexpr auto utf8_lead_byte_size(const unsigned char byte) -> std::uint8_t {
  if (byte < 0x80) {
    return 1;
  }
  if (byte >= 0xC2 && byte <= 0xDF) {
    return 2;
  }
  if (byte >= 0xE0 && byte <= 0xEF) {
    return 3;
  }
  if (byte >= 0xF0 && byte <= 0xF4) {
    return 4;
  }
  return 0;
}

constexpr auto is_utf8_tail(const unsigned char lead,
                            const std::size_t position,
                            const unsigned char byte) -> bool {
  const unsigned char lower{static_cast<unsigned char>(
      position > 1 ? 0x80
                   : (lead == 0xE0 ? 0xA0 : (lead == 0xF0 ? 0x90 : 0x80)))};
  const unsigned char upper{static_cast<unsigned char>(
      position > 1 ? 0xBF
                   : (lead == 0xED ? 0x9F : (lead == 0xF4 ? 0x8F : 0xBF)))};
  return byte >= lower && byte <= upper;
}

constexpr auto utf8_sequence_size(const std::string_view input) -> std::size_t {
  if (input.empty()) {
    return 0;
  }
 
  const auto lead{static_cast<unsigned char>(input.front())};
  const std::size_t size{utf8_lead_byte_size(lead)};
  if (size == 0 || input.size() < size) {
    return 0;
  }
 
  for (std::size_t index{1}; index < size; index += 1) {
    if (!is_utf8_tail(lead, index, static_cast<unsigned char>(input[index]))) {
      return 0;
    }
  }
 
  return size;
}

inline auto is_valid_utf8(const std::string_view input) noexcept -> bool {
  constexpr std::uint64_t HIGH_BITS{0x8080808080808080ULL};
  const auto size{input.size()};
  std::size_t position{0};
  while (position < size) {
    // Comparing what is left rather than the position past the word keeps the
    // sum of a position and a word from wrapping around
    if (size - position >= 8) {
      std::uint64_t word{0};
      std::memcpy(&word, input.data() + position, 8);
      if ((word & HIGH_BITS) == 0) {
        position += 8;
        continue;
      }
    }
 
    if (static_cast<unsigned char>(input[position]) < 0x80) {
      position += 1;
      continue;
    }
 
    const auto length{utf8_sequence_size(input.substr(position))};
    if (length == 0) {
      return false;
    }
 
    position += length;
  }
 
  return true;
}

constexpr auto is_utf8_continuation(const unsigned char byte) -> bool {
  return byte >= 0x80 && byte <= 0xBF;
}

constexpr auto utf8_codepoint_count(const std::string_view input)
    -> std::size_t {
  std::size_t count{0};
  for (const auto byte : input) {
    if (!is_utf8_continuation(static_cast<unsigned char>(byte))) {
      count += 1;
    }
  }
 
  return count;
}

constexpr auto utf8_codepoint_within(const std::string_view input,
                                     const std::size_t minimum,
                                     const std::size_t maximum) -> bool {
  const auto bytes{input.size()};
 
  // A code point is at least one byte, so the count never exceeds the byte
  // length: fewer bytes than the minimum cannot reach it
  if (bytes < minimum) {
    return false;
  }
 
  // A code point is at most four bytes, so the count is at least the byte
  // length divided by four (rounded up): too many bytes cannot fit the maximum
  const auto lower_bound{(bytes + 3) / 4};
  if (lower_bound > maximum) {
    return false;
  }
 
  // If the byte length already satisfies the maximum and the rounded-up lower
  // bound already satisfies the minimum, the count must be in range
  if (bytes <= maximum && lower_bound >= minimum) {
    return true;
  }
 
  // Otherwise count, stopping as soon as the maximum is exceeded. Reaching here
  // implies the byte length is at most four times the maximum, so this is
  // bounded regardless of how long the input is
  std::size_t count{0};
  for (const auto byte : input) {
    if (!is_utf8_continuation(static_cast<unsigned char>(byte))) {
      count += 1;
      if (count > maximum) {
        return false;
      }
    }
  }
 
  return count >= minimum;
}

constexpr auto is_surrogate(const char32_t codepoint) -> bool {
  return codepoint >= 0xD800 && codepoint <= 0xDFFF;
}

constexpr auto is_valid_codepoint(const char32_t codepoint) -> bool {
  return codepoint <= 0x10FFFF && !is_surrogate(codepoint);
}

constexpr auto is_ucschar(const char32_t codepoint) -> bool {
  if (codepoint >= 0xA0 && codepoint <= 0xD7FF) {
    return true;
  }
  if (codepoint >= 0xF900 && codepoint <= 0xFDCF) {
    return true;
  }
  if (codepoint >= 0xFDF0 && codepoint <= 0xFFEF) {
    return true;
  }
  // Supplementary planes 1 through 14. Each plane allows 0..FFFD;
  // FFFE and FFFF are noncharacters. Plane 14 starts at offset 0x1000
  // rather than 0x0000.
  if (codepoint < 0x10000 || codepoint > 0xEFFFD) {
    return false;
  }
  if (codepoint >= 0xE0000 && codepoint < 0xE1000) {
    return false;
  }
  return (codepoint & 0xFFFFU) <= 0xFFFDU;
}

constexpr auto is_iprivate(const char32_t codepoint) -> bool {
  if (codepoint >= 0xE000 && codepoint <= 0xF8FF) {
    return true;
  }
  if (codepoint >= 0xF0000 && codepoint <= 0xFFFFD) {
    return true;
  }
  if (codepoint >= 0x100000 && codepoint <= 0x10FFFD) {
    return true;
  }
  return false;
}

constexpr auto utf8_codepoint_byte_count(const char32_t codepoint)
    -> std::uint8_t {
  if (codepoint < 0x80) {
    return 1;
  }
  if (codepoint < 0x800) {
    return 2;
  }
  if (codepoint < 0x10000) {
    return 3;
  }
  return 4;
}

SOURCEMETA_CORE_UNICODE_EXPORT
auto combining_class(const char32_t codepoint) noexcept -> std::uint8_t;

SOURCEMETA_CORE_UNICODE_EXPORT
auto joining_type(const char32_t codepoint) noexcept -> JoiningType;

SOURCEMETA_CORE_UNICODE_EXPORT
auto bidi_class(const char32_t codepoint) noexcept -> BidiClass;

SOURCEMETA_CORE_UNICODE_EXPORT
auto script(const char32_t codepoint) noexcept -> UnicodeScript;

SOURCEMETA_CORE_UNICODE_EXPORT
auto general_category(const char32_t codepoint) noexcept -> GeneralCategory;

SOURCEMETA_CORE_UNICODE_EXPORT
auto is_combining_mark(const char32_t codepoint) noexcept -> bool;

SOURCEMETA_CORE_UNICODE_EXPORT
auto is_id_start(const char32_t codepoint) noexcept -> bool;

SOURCEMETA_CORE_UNICODE_EXPORT
auto is_id_continue(const char32_t codepoint) noexcept -> bool;

SOURCEMETA_CORE_UNICODE_EXPORT
auto nfc_quick_check(const char32_t codepoint) noexcept -> NFCQuickCheck;

SOURCEMETA_CORE_UNICODE_EXPORT
auto canonical_decomposition(const char32_t codepoint) noexcept
    -> std::u32string_view;

SOURCEMETA_CORE_UNICODE_EXPORT
auto canonical_composition(const char32_t starter,
                           const char32_t combining) noexcept
    -> std::optional<char32_t>;

SOURCEMETA_CORE_UNICODE_EXPORT
auto nfc(const std::u32string_view input) -> std::u32string;

SOURCEMETA_CORE_UNICODE_EXPORT
auto is_nfc(const std::u32string_view input) -> bool;

SOURCEMETA_CORE_UNICODE_EXPORT
auto case_fold(const char32_t codepoint) noexcept -> std::u32string_view;

SOURCEMETA_CORE_UNICODE_EXPORT
auto case_fold(const std::u32string_view input) -> std::u32string;

constexpr auto utf8_codepoint_length(const std::string_view input,
                                     const std::string_view::size_type position)
    -> std::size_t {
  if (position >= input.size()) {
    return 0;
  }
  const auto byte_0{static_cast<unsigned char>(input[position])};
  const auto size{utf8_lead_byte_size(byte_0)};
  if (size == 0 || position + size > input.size()) {
    return 0;
  }
  if (size == 1) {
    return 1;
  }
 
  // The second byte after the lead has tighter sub-ranges for specific leads
  // (RFC 3629 §4) that exclude overlong encodings, surrogates, and code
  // points above U+10FFFF
  const auto byte_1{static_cast<unsigned char>(input[position + 1])};
  bool byte_1_ok{false};
  if (size == 2) {
    byte_1_ok = is_utf8_continuation(byte_1);
  } else if (size == 3) {
    if (byte_0 == 0xE0) {
      byte_1_ok = byte_1 >= 0xA0 && byte_1 <= 0xBF;
    } else if (byte_0 == 0xED) {
      byte_1_ok = byte_1 >= 0x80 && byte_1 <= 0x9F;
    } else {
      byte_1_ok = is_utf8_continuation(byte_1);
    }
  } else {
    if (byte_0 == 0xF0) {
      byte_1_ok = byte_1 >= 0x90 && byte_1 <= 0xBF;
    } else if (byte_0 == 0xF4) {
      byte_1_ok = byte_1 >= 0x80 && byte_1 <= 0x8F;
    } else {
      byte_1_ok = is_utf8_continuation(byte_1);
    }
  }
 
  if (!byte_1_ok) {
    return 0;
  }
 
  // Remaining continuation bytes (if any) are unconstrained beyond the
  // continuation byte range
  for (std::size_t index{2}; index < size; ++index) {
    if (!is_utf8_continuation(
            static_cast<unsigned char>(input[position + index]))) {
      return 0;
    }
  }
 
  return size;
}

constexpr auto utf8_decode(const std::string_view input,
                           const std::string_view::size_type position)
    -> std::optional<std::pair<char32_t, std::size_t>> {
  const auto size{utf8_codepoint_length(input, position)};
  if (size == 0) {
    return std::nullopt;
  }
 
  const auto lead{static_cast<unsigned char>(input[position])};
  char32_t codepoint{0};
  if (size == 1) {
    codepoint = static_cast<char32_t>(lead);
  } else if (size == 2) {
    codepoint = static_cast<char32_t>(lead & 0x1FU);
  } else if (size == 3) {
    codepoint = static_cast<char32_t>(lead & 0x0FU);
  } else {
    codepoint = static_cast<char32_t>(lead & 0x07U);
  }
 
  for (std::size_t index{1}; index < size; ++index) {
    const auto continuation{
        static_cast<unsigned char>(input[position + index])};
    codepoint = (codepoint << 6) | static_cast<char32_t>(continuation & 0x3FU);
  }
 
  return std::make_pair(codepoint, size);
}
 
} // namespace sourcemeta::core
 
#endif