utf8.ipp 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379
  1. // Copyright (c) 2022 Klemens D. Morgenstern
  2. //
  3. // Distributed under the Boost Software License, Version 1.0. (See accompanying
  4. // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
  5. #ifndef BOOST_PROCESS_V2_DETAIL_IMPL_UTF8_HPP
  6. #define BOOST_PROCESS_V2_DETAIL_IMPL_UTF8_HPP
  7. #include <boost/process/v2/detail/utf8.hpp>
  8. #include <boost/process/v2/detail/config.hpp>
  9. #include <boost/process/v2/detail/last_error.hpp>
  10. #include <boost/process/v2/error.hpp>
  11. #if defined(BOOST_PROCESS_V2_WINDOWS)
  12. #include <Windows.h>
  13. #endif
  14. BOOST_PROCESS_V2_BEGIN_NAMESPACE
  15. namespace detail
  16. {
  17. #if defined(BOOST_PROCESS_V2_WINDOWS)
  18. inline void handle_error(error_code & ec)
  19. {
  20. const auto err = ::GetLastError();
  21. switch (err)
  22. {
  23. case ERROR_INSUFFICIENT_BUFFER:
  24. BOOST_PROCESS_V2_ASSIGN_EC(ec, error::insufficient_buffer, error::utf8_category)
  25. break;
  26. case ERROR_NO_UNICODE_TRANSLATION:
  27. BOOST_PROCESS_V2_ASSIGN_EC(ec, error::invalid_character, error::utf8_category)
  28. break;
  29. default:
  30. BOOST_PROCESS_V2_ASSIGN_EC(ec, err, system_category())
  31. }
  32. }
  33. std::size_t size_as_utf8(const wchar_t * in, std::size_t size, error_code & ec)
  34. {
  35. auto res = WideCharToMultiByte(
  36. CP_UTF8, // CodePage,
  37. 0, // dwFlags,
  38. in, // lpWideCharStr,
  39. static_cast<int>(size), // cchWideChar,
  40. nullptr, // lpMultiByteStr,
  41. 0, // cbMultiByte,
  42. nullptr, // lpDefaultChar,
  43. FALSE); // lpUsedDefaultChar
  44. if (res == 0u)
  45. handle_error(ec);
  46. return static_cast<std::size_t>(res);
  47. }
  48. std::size_t size_as_wide(const char * in, std::size_t size, error_code & ec)
  49. {
  50. auto res = ::MultiByteToWideChar(
  51. CP_UTF8, // CodePage
  52. 0, // dwFlags
  53. in, // lpMultiByteStr
  54. static_cast<int>(size), // cbMultiByte
  55. nullptr, // lpWideCharStr
  56. 0); // cchWideChar
  57. if (res == 0u)
  58. handle_error(ec);
  59. return static_cast<std::size_t>(res);
  60. }
  61. std::size_t convert_to_utf8(const wchar_t *in, std::size_t size, char * out,
  62. std::size_t max_size, error_code & ec)
  63. {
  64. auto res = ::WideCharToMultiByte(
  65. CP_UTF8, // CodePage
  66. 0, // dwFlags
  67. in, // lpWideCharStr
  68. static_cast<int>(size), // cchWideChar
  69. out, // lpMultiByteStr
  70. static_cast<int>(max_size), // cbMultiByte
  71. nullptr, // lpDefaultChar
  72. FALSE); // lpUsedDefaultChar
  73. if (res == 0u)
  74. handle_error(ec);
  75. return static_cast<std::size_t>(res);
  76. }
  77. std::size_t convert_to_wide(const char *in, std::size_t size, wchar_t * out,
  78. std::size_t max_size, error_code & ec)
  79. {
  80. auto res = ::MultiByteToWideChar(
  81. CP_UTF8, // CodePage
  82. 0, // dwFlags
  83. in, // lpMultiByteStr
  84. static_cast<int>(size), // cbMultiByte
  85. out, // lpWideCharStr
  86. static_cast<int>(max_size)); // cchWideChar
  87. if (res == 0u)
  88. handle_error(ec);
  89. return static_cast<std::size_t>(res);
  90. }
  91. #else
  92. template<std::size_t s>
  93. inline int get_cont_octet_out_count_impl(wchar_t word) {
  94. if (word < 0x80) {
  95. return 0;
  96. }
  97. if (word < 0x800) {
  98. return 1;
  99. }
  100. return 2;
  101. }
  102. template<>
  103. inline int get_cont_octet_out_count_impl<4>(wchar_t word) {
  104. if (word < 0x80) {
  105. return 0;
  106. }
  107. if (word < 0x800) {
  108. return 1;
  109. }
  110. // Note that the following code will generate warnings on some platforms
  111. // where wchar_t is defined as UCS2. The warnings are superfluous as the
  112. // specialization is never instantiated with such compilers, but this
  113. // can cause problems if warnings are being treated as errors, so we guard
  114. // against that. Including <boost/detail/utf8_codecvt_facet.hpp> as we do
  115. // should be enough to get WCHAR_MAX defined.
  116. #if !defined(WCHAR_MAX)
  117. # error WCHAR_MAX not defined!
  118. #endif
  119. // cope with VC++ 7.1 or earlier having invalid WCHAR_MAX
  120. #if defined(_MSC_VER) && _MSC_VER <= 1310 // 7.1 or earlier
  121. return 2;
  122. #elif WCHAR_MAX > 0x10000
  123. if (word < 0x10000) {
  124. return 2;
  125. }
  126. if (word < 0x200000) {
  127. return 3;
  128. }
  129. if (word < 0x4000000) {
  130. return 4;
  131. }
  132. return 5;
  133. #else
  134. return 2;
  135. #endif
  136. }
  137. inline int get_cont_octet_out_count(wchar_t word)
  138. {
  139. return detail::get_cont_octet_out_count_impl<sizeof(wchar_t)>(word);
  140. }
  141. // copied from boost/detail/utf8_codecvt_facet.ipp
  142. // Copyright (c) 2001 Ronald Garcia, Indiana University (garcia@osl.iu.edu)
  143. // Andrew Lumsdaine, Indiana University (lums@osl.iu.edu).
  144. inline unsigned int get_octet_count(unsigned char lead_octet)
  145. {
  146. // if the 0-bit (MSB) is 0, then 1 character
  147. if (lead_octet <= 0x7f) return 1;
  148. // Otherwise the count number of consecutive 1 bits starting at MSB
  149. // assert(0xc0 <= lead_octet && lead_octet <= 0xfd);
  150. if (0xc0 <= lead_octet && lead_octet <= 0xdf) return 2;
  151. else if (0xe0 <= lead_octet && lead_octet <= 0xef) return 3;
  152. else if (0xf0 <= lead_octet && lead_octet <= 0xf7) return 4;
  153. else if (0xf8 <= lead_octet && lead_octet <= 0xfb) return 5;
  154. else return 6;
  155. }
  156. inline bool invalid_continuing_octet(unsigned char octet_1) {
  157. return (octet_1 < 0x80|| 0xbf< octet_1);
  158. }
  159. inline unsigned int get_cont_octet_count(unsigned char lead_octet)
  160. {
  161. return get_octet_count(lead_octet) - 1;
  162. }
  163. inline const wchar_t * get_octet1_modifier_table() noexcept
  164. {
  165. static const wchar_t octet1_modifier_table[] = {
  166. 0x00, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc
  167. };
  168. return octet1_modifier_table;
  169. }
  170. std::size_t size_as_utf8(const wchar_t * in, std::size_t size, error_code & ec)
  171. {
  172. std::size_t res = 0u;
  173. const auto from_end = in + size;
  174. for (auto from = in; from != from_end; from++)
  175. res += get_cont_octet_out_count(*from) + 1;
  176. return res;
  177. }
  178. std::size_t size_as_wide(const char * in, std::size_t size, error_code & ec)
  179. {
  180. const auto from = in;
  181. const auto from_end = from + size;
  182. const char * from_next = from;
  183. for (std::size_t char_count = 0u; from_next < from_end; ++char_count) {
  184. unsigned int octet_count = get_octet_count(*from_next);
  185. // The buffer may represent incomplete characters, so terminate early if one is found
  186. if (octet_count > static_cast<std::size_t>(from_end - from_next))
  187. break;
  188. from_next += octet_count;
  189. }
  190. return from_next - from;
  191. }
  192. std::size_t convert_to_utf8(const wchar_t * in, std::size_t size,
  193. char * out, std::size_t max_size, error_code & ec)
  194. {
  195. const wchar_t * from = in;
  196. const wchar_t * from_end = from + size;
  197. const wchar_t * & from_next = from;
  198. char * to = out;
  199. char * to_end = out + max_size;
  200. char * & to_next = to;
  201. const wchar_t * const octet1_modifier_table = get_octet1_modifier_table();
  202. wchar_t max_wchar = (std::numeric_limits<wchar_t>::max)();
  203. while (from != from_end && to != to_end) {
  204. // Check for invalid UCS-4 character
  205. if (*from > max_wchar) {
  206. from_next = from;
  207. to_next = to;
  208. BOOST_PROCESS_V2_ASSIGN_EC(ec, error::invalid_character, error::get_utf8_category())
  209. return 0u;
  210. }
  211. int cont_octet_count = get_cont_octet_out_count(*from);
  212. // RG - comment this formula better
  213. int shift_exponent = cont_octet_count * 6;
  214. // Process the first character
  215. *to++ = static_cast<char>(octet1_modifier_table[cont_octet_count] +
  216. (unsigned char)(*from / (1 << shift_exponent)));
  217. // Process the continuation characters
  218. // Invariants: At the start of the loop:
  219. // 1) 'i' continuing octets have been generated
  220. // 2) '*to' points to the next location to place an octet
  221. // 3) shift_exponent is 6 more than needed for the next octet
  222. int i = 0;
  223. while (i != cont_octet_count && to != to_end) {
  224. shift_exponent -= 6;
  225. *to++ = static_cast<char>(0x80 + ((*from / (1 << shift_exponent)) % (1 << 6)));
  226. ++i;
  227. }
  228. // If we filled up the out buffer before encoding the character
  229. if (to == to_end && i != cont_octet_count) {
  230. from_next = from;
  231. to_next = to - (i + 1);
  232. BOOST_PROCESS_V2_ASSIGN_EC(ec, error::insufficient_buffer, error::get_utf8_category())
  233. return 0u;
  234. }
  235. ++from;
  236. }
  237. from_next = from;
  238. to_next = to;
  239. // Were we done or did we run out of destination space
  240. if (from != from_end)
  241. BOOST_PROCESS_V2_ASSIGN_EC(ec, error::insufficient_buffer, error::get_utf8_category())
  242. return to_next - out;
  243. }
  244. inline bool invalid_leading_octet(unsigned char octet_1) {
  245. return (0x7f < octet_1 && octet_1 < 0xc0) ||
  246. (octet_1 > 0xfd);
  247. }
  248. std::size_t convert_to_wide(const char * in, std::size_t size,
  249. wchar_t * out, std::size_t max_size, error_code & ec)
  250. {
  251. const char * from = in;
  252. const char * from_end = from + size;
  253. const char * & from_next = from;
  254. wchar_t * to = out;
  255. wchar_t * to_end = out + max_size;
  256. wchar_t * & to_next = to;
  257. // Basic algorithm: The first octet determines how many
  258. // octets total make up the UCS-4 character. The remaining
  259. // "continuing octets" all begin with "10". To convert, subtract
  260. // the amount that specifies the number of octets from the first
  261. // octet. Subtract 0x80 (1000 0000) from each continuing octet,
  262. // then mash the whole lot together. Note that each continuing
  263. // octet only uses 6 bits as unique values, so only shift by
  264. // multiples of 6 to combine.
  265. const wchar_t * const octet1_modifier_table = detail::get_octet1_modifier_table();
  266. while (from != from_end && to != to_end) {
  267. // Error checking on the first octet
  268. if (invalid_leading_octet(*from)) {
  269. from_next = from;
  270. to_next = to;
  271. BOOST_PROCESS_V2_ASSIGN_EC(ec, error::invalid_character, error::get_utf8_category())
  272. return 0u;
  273. }
  274. // The first octet is adjusted by a value dependent upon
  275. // the number of "continuing octets" encoding the character
  276. const int cont_octet_count = get_cont_octet_count(*from);
  277. // The unsigned char conversion is necessary in case char is
  278. // signed (I learned this the hard way)
  279. wchar_t ucs_result =
  280. (unsigned char)(*from++) - octet1_modifier_table[cont_octet_count];
  281. // Invariants:
  282. // 1) At the start of the loop, 'i' continuing characters have been
  283. // processed
  284. // 2) *from points to the next continuing character to be processed.
  285. int i = 0;
  286. while (i != cont_octet_count && from != from_end) {
  287. // Error checking on continuing characters
  288. if (invalid_continuing_octet(*from)) {
  289. from_next = from;
  290. to_next = to;
  291. BOOST_PROCESS_V2_ASSIGN_EC(ec, error::invalid_character, error::get_utf8_category())
  292. return 0u;
  293. }
  294. ucs_result *= (1 << 6);
  295. // each continuing character has an extra (10xxxxxx)b attached to
  296. // it that must be removed.
  297. ucs_result += (unsigned char)(*from++) - 0x80;
  298. ++i;
  299. }
  300. // If the buffer ends with an incomplete unicode character...
  301. if (from == from_end && i != cont_octet_count) {
  302. // rewind "from" to before the current character translation
  303. from_next = from - (i + 1);
  304. to_next = to;
  305. BOOST_PROCESS_V2_ASSIGN_EC(ec, error::insufficient_buffer, error::get_utf8_category())
  306. return 0u;
  307. }
  308. *to++ = ucs_result;
  309. }
  310. from_next = from;
  311. to_next = to;
  312. if (from != from_end)
  313. BOOST_PROCESS_V2_ASSIGN_EC(ec, error::insufficient_buffer, error::get_utf8_category())
  314. return to_next - out;
  315. }
  316. #endif
  317. }
  318. BOOST_PROCESS_V2_END_NAMESPACE
  319. #endif //BOOST_PROCESS_V2_DETAIL_IMPL_UTF8_HPP