diff options
| author | 2017-01-01 12:08:40 -0500 | |
|---|---|---|
| committer | 2017-01-01 12:09:38 -0500 | |
| commit | 34048b54f64946e9669a0963560a7f732f8d3169 (patch) | |
| tree | 4f918b50010d4b414aac74e5fff68986a57cdd6b /src/tools | |
| parent | 2679c0f553ea270e058c4c2b1ee1f0788c8618d5 (diff) | |
Updated to support Unicode normalization and simple charconverters that alter the low page
Diffstat (limited to 'src/tools')
| -rw-r--r-- | src/tools/imgtool/charconv.cpp | 64 | ||||
| -rw-r--r-- | src/tools/imgtool/charconv.h | 13 |
2 files changed, 57 insertions, 20 deletions
diff --git a/src/tools/imgtool/charconv.cpp b/src/tools/imgtool/charconv.cpp index 89c5f01af56..dbe612876ff 100644 --- a/src/tools/imgtool/charconv.cpp +++ b/src/tools/imgtool/charconv.cpp @@ -33,7 +33,7 @@ static const char32_t iso_8859_1_code_page[128] = 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF }; -imgtool::simple_charconverter imgtool::charconverter_iso_8859_1(iso_8859_1_code_page); +imgtool::simple_charconverter imgtool::charconverter_iso_8859_1(nullptr, iso_8859_1_code_page); //------------------------------------------------- @@ -42,26 +42,49 @@ imgtool::simple_charconverter imgtool::charconverter_iso_8859_1(iso_8859_1_code_ void imgtool::simple_charconverter::from_utf8(std::ostream &dest, const std::string &src) const { - const util::contiguous_sequence_wrapper<const char32_t> lookup(m_highpage, 0x80); + // normalize the incoming unicode + std::string normalized_src = normalize_unicode(src, m_norm); - auto iter = src.begin(); - while(iter != src.end()) + auto iter = normalized_src.begin(); + while(iter != normalized_src.end()) { + // get the next character char32_t ch; - iter += uchar_from_utf8(&ch, &*iter, src.end() - iter); + iter += uchar_from_utf8(&ch, &*iter, normalized_src.end() - iter); - if (ch <= 0x7F) - { - dest << (char)ch; - } - else - { - auto lookup_iter = std::find(lookup.begin(), lookup.end(), ch); - if (lookup_iter == lookup.end()) - throw charconverter_exception(); + // look in all pages + const char32_t *pages[2]; + pages[0] = m_lowpage; + pages[1] = m_highpage; - dest << (char)(0x80 + (lookup_iter - lookup.begin())); + bool found = false; + for (int i = 0; !found && i < ARRAY_LENGTH(pages); i++) + { + if (pages[i] == nullptr) + { + // null page; perhaps we can just emit this + if (ch >= i * 0x80 && (ch < (i + 1) * 0x80)) + { + dest << (char)ch; + found = true; + } + } + else + { + // non-null page; perform a lookup + // if we have a page, perform the lookup + const util::contiguous_sequence_wrapper<const char32_t> lookup(pages[i], 0x80); + auto lookup_iter = std::find(lookup.begin(), lookup.end(), ch); + if (lookup_iter != lookup.end()) + { + // and emit the result + dest << (char)((i * 0x80) + (lookup_iter - lookup.begin())); + found = true; + } + } } + if (!found) + throw charconverter_exception(); } } @@ -74,15 +97,20 @@ void imgtool::simple_charconverter::to_utf8(std::ostream &dest, const std::strin { for (auto iter = src.begin(); iter != src.end(); iter++) { + // which page is this in? + const char32_t *page = ((*iter & 0x80) == 0) ? m_lowpage : m_highpage; + + // is this page present? if ((*iter & 0x80) == 0) { - // low page (0x00 - 0x7F) - pass it on + // no - pass it on dest << *iter; } else { - // high page (0x80 - 0xFF) - we need to do a lookup - char32_t ch = m_highpage[((unsigned char)(*iter)) - 0x80]; + // yes - we need to do a lookup + size_t base = ((*iter & 0x80) == 0) ? 0x00 : 0x80; + char32_t ch = page[((unsigned char)(*iter)) - base]; if (ch == 0) throw charconverter_exception(); diff --git a/src/tools/imgtool/charconv.h b/src/tools/imgtool/charconv.h index e2abb3ecd57..03ea584a24c 100644 --- a/src/tools/imgtool/charconv.h +++ b/src/tools/imgtool/charconv.h @@ -12,6 +12,8 @@ #ifndef IMGTOOL_CHARCONV_H #define IMGTOOL_CHARCONV_H +#include "unicode.h" + namespace imgtool { // ======================> charconverter @@ -47,8 +49,13 @@ namespace imgtool class simple_charconverter : public charconverter { public: - constexpr simple_charconverter(const char32_t highpage[0x80]) - : m_highpage(highpage) + constexpr simple_charconverter(const char32_t highpage[0x80], unicode_normalization_form norm = unicode_normalization_form::C) + : m_norm(norm), m_lowpage(nullptr), m_highpage(highpage) + { + } + + constexpr simple_charconverter(const char32_t lowpage[0x80], const char32_t highpage[0x80], unicode_normalization_form norm = unicode_normalization_form::C) + : m_norm(norm), m_lowpage(lowpage), m_highpage(highpage) { } @@ -56,6 +63,8 @@ namespace imgtool virtual void to_utf8(std::ostream &dest, const std::string &src) const override; private: + unicode_normalization_form m_norm; + const char32_t *m_lowpage; const char32_t *m_highpage; }; |
