123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472 |
- #ifndef BOOST_LOCALE_GENERIC_CODECVT_HPP
- #define BOOST_LOCALE_GENERIC_CODECVT_HPP
- #include <boost/locale/utf.hpp>
- #include <cstdint>
- #include <locale>
- namespace boost { namespace locale {
- static_assert(sizeof(std::mbstate_t) >= 2, "std::mbstate_t is to small to store an UTF-16 codepoint");
- namespace detail {
-
- inline void copy_uint16_t(void* dst, const void* src)
- {
- unsigned char* cdst = static_cast<unsigned char*>(dst);
- const unsigned char* csrc = static_cast<const unsigned char*>(src);
- cdst[0] = csrc[0];
- cdst[1] = csrc[1];
- }
- inline uint16_t read_state(const std::mbstate_t& src)
- {
- uint16_t dst;
- copy_uint16_t(&dst, &src);
- return dst;
- }
- inline void write_state(std::mbstate_t& dst, const uint16_t src)
- {
- copy_uint16_t(&dst, &src);
- }
- }
-
- class generic_codecvt_base {
- public:
-
- enum initial_convertion_state {
- to_unicode_state,
- from_unicode_state
- };
- };
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- template<typename CharType, typename CodecvtImpl, int CharSize = sizeof(CharType)>
- class generic_codecvt;
-
-
-
-
-
-
- template<typename CharType, typename CodecvtImpl>
- class generic_codecvt<CharType, CodecvtImpl, 2> : public std::codecvt<CharType, char, std::mbstate_t>,
- public generic_codecvt_base {
- public:
- typedef CharType uchar;
- generic_codecvt(size_t refs = 0) : std::codecvt<CharType, char, std::mbstate_t>(refs) {}
- const CodecvtImpl& implementation() const { return *static_cast<const CodecvtImpl*>(this); }
- protected:
- std::codecvt_base::result do_unshift(std::mbstate_t& s, char* from, char* , char*& next) const override
- {
- if(*reinterpret_cast<char*>(&s) != 0)
- return std::codecvt_base::error;
- next = from;
- return std::codecvt_base::ok;
- }
- int do_encoding() const noexcept override { return 0; }
- int do_max_length() const noexcept override { return implementation().max_encoding_length(); }
- bool do_always_noconv() const noexcept override { return false; }
- int do_length(std::mbstate_t& std_state, const char* from, const char* from_end, size_t max) const override
- {
- bool state = *reinterpret_cast<char*>(&std_state) != 0;
- const char* save_from = from;
- auto cvt_state = implementation().initial_state(to_unicode_state);
- while(max > 0 && from < from_end) {
- const char* prev_from = from;
- const utf::code_point ch = implementation().to_unicode(cvt_state, from, from_end);
- if(ch == boost::locale::utf::incomplete || ch == boost::locale::utf::illegal) {
- from = prev_from;
- break;
- }
- max--;
- if(ch > 0xFFFF) {
- if(!state)
- from = prev_from;
- state = !state;
- }
- }
- *reinterpret_cast<char*>(&std_state) = state;
- return static_cast<int>(from - save_from);
- }
- std::codecvt_base::result do_in(std::mbstate_t& std_state,
- const char* from,
- const char* from_end,
- const char*& from_next,
- uchar* to,
- uchar* to_end,
- uchar*& to_next) const override
- {
- std::codecvt_base::result r = std::codecvt_base::ok;
-
-
-
-
-
- bool state = *reinterpret_cast<char*>(&std_state) != 0;
- auto cvt_state = implementation().initial_state(to_unicode_state);
- while(to < to_end && from < from_end) {
- const char* from_saved = from;
- utf::code_point ch = implementation().to_unicode(cvt_state, from, from_end);
- if(ch == boost::locale::utf::illegal) {
- from = from_saved;
- r = std::codecvt_base::error;
- break;
- }
- if(ch == boost::locale::utf::incomplete) {
- from = from_saved;
- r = std::codecvt_base::partial;
- break;
- }
-
- if(ch <= 0xFFFF)
- *to++ = static_cast<uchar>(ch);
- else {
-
-
-
-
-
-
-
-
-
- ch -= 0x10000;
- std::uint16_t w1 = static_cast<std::uint16_t>(0xD800 | (ch >> 10));
- std::uint16_t w2 = static_cast<std::uint16_t>(0xDC00 | (ch & 0x3FF));
- if(!state) {
- from = from_saved;
- *to++ = w1;
- } else
- *to++ = w2;
- state = !state;
- }
- }
- from_next = from;
- to_next = to;
- if(r == std::codecvt_base::ok && (from != from_end || state))
- r = std::codecvt_base::partial;
- *reinterpret_cast<char*>(&std_state) = state;
- return r;
- }
- std::codecvt_base::result do_out(std::mbstate_t& std_state,
- const uchar* from,
- const uchar* from_end,
- const uchar*& from_next,
- char* to,
- char* to_end,
- char*& to_next) const override
- {
- std::codecvt_base::result r = std::codecvt_base::ok;
-
-
-
-
-
-
- std::uint16_t state = detail::read_state(std_state);
- auto cvt_state = implementation().initial_state(from_unicode_state);
- while(to < to_end && from < from_end) {
- utf::code_point ch = 0;
- if(state != 0) {
-
-
-
- std::uint16_t w1 = state;
- std::uint16_t w2 = *from;
-
-
- if(0xDC00 <= w2 && w2 <= 0xDFFF) {
- std::uint16_t vh = w1 - 0xD800;
- std::uint16_t vl = w2 - 0xDC00;
- ch = ((uint32_t(vh) << 10) | vl) + 0x10000;
- } else {
-
- r = std::codecvt_base::error;
- break;
- }
- } else {
- ch = *from;
- if(0xD800 <= ch && ch <= 0xDBFF) {
-
-
-
-
- state = static_cast<uint16_t>(ch);
- from++;
- continue;
- } else if(0xDC00 <= ch && ch <= 0xDFFF) {
-
-
-
- r = std::codecvt_base::error;
- break;
- }
- }
- if(!boost::locale::utf::is_valid_codepoint(ch)) {
- r = std::codecvt_base::error;
- break;
- }
- const utf::code_point len = implementation().from_unicode(cvt_state, ch, to, to_end);
- if(len == boost::locale::utf::incomplete) {
- r = std::codecvt_base::partial;
- break;
- } else if(len == boost::locale::utf::illegal) {
- r = std::codecvt_base::error;
- break;
- } else
- to += len;
- state = 0;
- from++;
- }
- from_next = from;
- to_next = to;
- if(r == std::codecvt_base::ok && (from != from_end || state != 0))
- r = std::codecvt_base::partial;
- detail::write_state(std_state, state);
- return r;
- }
- };
-
-
-
-
- template<typename CharType, typename CodecvtImpl>
- class generic_codecvt<CharType, CodecvtImpl, 4> : public std::codecvt<CharType, char, std::mbstate_t>,
- public generic_codecvt_base {
- public:
- typedef CharType uchar;
- generic_codecvt(size_t refs = 0) : std::codecvt<CharType, char, std::mbstate_t>(refs) {}
- const CodecvtImpl& implementation() const { return *static_cast<const CodecvtImpl*>(this); }
- protected:
- std::codecvt_base::result
- do_unshift(std::mbstate_t& , char* from, char* , char*& next) const override
- {
- next = from;
- return std::codecvt_base::ok;
- }
- int do_encoding() const noexcept override { return 0; }
- int do_max_length() const noexcept override { return implementation().max_encoding_length(); }
- bool do_always_noconv() const noexcept override { return false; }
- int do_length(std::mbstate_t& , const char* from, const char* from_end, size_t max) const override
- {
- const char* start_from = from;
- auto cvt_state = implementation().initial_state(to_unicode_state);
- while(max > 0 && from < from_end) {
- const char* save_from = from;
- const utf::code_point ch = implementation().to_unicode(cvt_state, from, from_end);
- if(ch == boost::locale::utf::incomplete || ch == boost::locale::utf::illegal) {
- from = save_from;
- break;
- }
- max--;
- }
- return static_cast<int>(from - start_from);
- }
- std::codecvt_base::result do_in(std::mbstate_t& ,
- const char* from,
- const char* from_end,
- const char*& from_next,
- uchar* to,
- uchar* to_end,
- uchar*& to_next) const override
- {
- std::codecvt_base::result r = std::codecvt_base::ok;
- auto cvt_state = implementation().initial_state(to_unicode_state);
- while(to < to_end && from < from_end) {
- const char* from_saved = from;
- const utf::code_point ch = implementation().to_unicode(cvt_state, from, from_end);
- if(ch == boost::locale::utf::illegal) {
- r = std::codecvt_base::error;
- from = from_saved;
- break;
- }
- if(ch == boost::locale::utf::incomplete) {
- r = std::codecvt_base::partial;
- from = from_saved;
- break;
- }
- *to++ = ch;
- }
- from_next = from;
- to_next = to;
- if(r == std::codecvt_base::ok && from != from_end)
- r = std::codecvt_base::partial;
- return r;
- }
- std::codecvt_base::result do_out(std::mbstate_t& ,
- const uchar* from,
- const uchar* from_end,
- const uchar*& from_next,
- char* to,
- char* to_end,
- char*& to_next) const override
- {
- std::codecvt_base::result r = std::codecvt_base::ok;
- auto cvt_state = implementation().initial_state(from_unicode_state);
- while(to < to_end && from < from_end) {
- const std::uint32_t ch = *from;
- if(!boost::locale::utf::is_valid_codepoint(ch)) {
- r = std::codecvt_base::error;
- break;
- }
- const utf::code_point len = implementation().from_unicode(cvt_state, ch, to, to_end);
- if(len == boost::locale::utf::incomplete) {
- r = std::codecvt_base::partial;
- break;
- } else if(len == boost::locale::utf::illegal) {
- r = std::codecvt_base::error;
- break;
- }
- to += len;
- from++;
- }
- from_next = from;
- to_next = to;
- if(r == std::codecvt_base::ok && from != from_end)
- r = std::codecvt_base::partial;
- return r;
- }
- };
- template<typename CodecvtImpl>
- class generic_codecvt<char, CodecvtImpl, 1> : public std::codecvt<char, char, std::mbstate_t>,
- public generic_codecvt_base {
- public:
- typedef char uchar;
- const CodecvtImpl& implementation() const { return *static_cast<const CodecvtImpl*>(this); }
- generic_codecvt(size_t refs = 0) : std::codecvt<char, char, std::mbstate_t>(refs) {}
- };
- }}
- #endif
|