C++ encoding conversion

/*
 * Copyright 2013 Florian Philipp
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0

 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include <locale>
// using std::locale, std::codecvt
#include <cwchar>
// using std::mbstate_t
#include <string>
// using std::string
#include <cstddef>
// using EXIT_SUCCESS
#include <stdexcept>
// using std::domain_error, std::runtime_error
#include <vector>
// using std::vector
#include <cstring>
// using std::strlen
#include <iterator>
// using std::distance
#include <sstream>
// using std::ostringstream, std::hex, std::endl
#include <iostream>
// using std::cout
#include <memory>
// using std::unique_ptr

namespace conv {
  namespace internal {
    /**
     * Shorthand notation for the conversion facet
     */
    typedef std::codecvt<wchar_t, char, std::mbstate_t> cvt_t;

    /**
     * Base class for EncodingState
     */
    class EncodingStateBase
    {
      const std::locale locale;
      std::mbstate_t state;
      cvt_t::result result;
    protected:
      EncodingStateBase(const std::locale& locale):
    locale(locale),
    state(std::mbstate_t() /* needs explicit default initialization */)
      {}
      ~EncodingStateBase() {}
      void set_result(cvt_t::result result) { this->result = result; }
      /**
       * One of two overloaded functions dispatching to cvt_t::in or cvt_t::out
       * depending on the parameters
       *
       * Simplifies the implementation of EncodingState::encode
       */
      static cvt_t::result do_convert(const cvt_t& cvt, std::mbstate_t& state,
                      const char* in, const char* in_end,
                      const char*& in_mark, wchar_t* out,
                      wchar_t* out_end, wchar_t*& out_mark)
      {
    return cvt.in(state, in, in_end, in_mark, out, out_end, out_mark);
      }
      static cvt_t::result do_convert(const cvt_t& cvt, std::mbstate_t& state,
                      const wchar_t* in, const wchar_t* in_end,
                      const wchar_t*& in_mark, char* out,
                      char* out_end, char*& out_mark)
      {
    return cvt.out(state, in, in_end, in_mark, out, out_end, out_mark);
      }
    public:
      /**
       * \return the locale used for the conversion
       */
      const std::locale& get_locale() const { return locale; }
      /**
       * \return a reference to the conversion state
       */
      std::mbstate_t& get_mbstate() { return state; }
      /**
       * \return the result of the last conversion. Undefined if encode has not
       * yet been called
       */
      cvt_t::result get_result() const { return result; }
    };

    /**
     * Keeps track of information relevant for an ongoing conversion between
     * wide and narrow characters
     *
     * \tparam source_char_t const char or const wchar_t
     * \tparam target_char_t char or wchar_t
     */
    template<class source_char_t, class target_char_t>
    class EncodingState: public EncodingStateBase
    {
    private:
      source_char_t* first_unconverted;
      source_char_t* const end_unconverted;
      target_char_t* target_buf;
      target_char_t* const end_target_buf;
    public:
      /**
       * Initializes the encoding state
       *
       * \param locale the encoding used for conversion. Needs to have the cvt_t
       * facet
       * \param source_begin first character to convert
       * \param source_end pointer behind the last character to convert
       * \param target_begin pointer to an output buffer. Needs to be large
       * enough
       * \param target_end pointer to the end of the output buffer
       */
      EncodingState(const std::locale& locale, source_char_t* source_begin,
            source_char_t* source_end, target_char_t* target_begin,
            target_char_t* target_end):
    EncodingStateBase(locale), first_unconverted(source_begin),
    end_unconverted(source_end), target_buf(target_begin),
    end_target_buf(target_end)
      {}
      /**
       * \return a pointer to the first character that has not been converted,
       * yet
       */
      source_char_t* get_first_unconverted() const { return first_unconverted; }
      /**
       * \return a pointer to the end of the input
       */
      source_char_t* get_end_unconverted() const { return end_unconverted; }
      /**
       * \return a pointer to the first unused character in the output buffer
       */
      target_char_t* get_target_buf() const { return target_buf; }
      /**
       * \return a pointer to the end of the output buffer
       */
      target_char_t* get_end_target() const { return end_target_buf; }
      /**
       * Skips the next input character, adding replacement to the output
       *
       * No range checks are performed prior to inserting the replacement into
       * the output buffer
       *
       * \param replacement a character in the output encoding
       */
      void replace_first(target_char_t replacement)
      {
    ++first_unconverted;
    *(target_buf++) = replacement;
      }
      /**
       * Converts using the current state
       *
       * Updates first_unconverted, target_buf, mbstate and result
       * \throw std::bad_cast if the locale does not have the cvt_t facet
       */
      void encode()
      {
    const cvt_t& converter = std::use_facet<cvt_t>(get_locale());
    source_char_t* source_mark;
    target_char_t* target_mark;
    set_result(do_convert(converter, get_mbstate(),
                  first_unconverted, end_unconverted, source_mark,
                  target_buf, end_target_buf, target_mark));
    first_unconverted = source_mark;
    target_buf = target_mark;
      }
    };

    /**
     * Instanziation of EncodingState for conversions from narrow characters
     * to wide characters
     */
    typedef EncodingState<const char, wchar_t> WidenEncodingState;

    /**
     * Instanziation of EncodingState for conversions from wide characters
     * to narrow characters
     */
    typedef EncodingState<const wchar_t, char> NarrowEncodingState;

    /**
     * Interface class of the strategy pattern to determine the way an
     * EncodingConverter reacts to conversion errors
     */
    class EncodingErrorStrategy
    {
    public:
      virtual ~EncodingErrorStrategy() {}
      /**
       * Evaluates and possibly updates the EncodingState after an encoding
       * attempt
       *
       * \param state the current conversion state, including a valid result
       * \return true if conversion shall be attempted again with the same
       * (updated) state
       * \throw any std::exception, depending on the specialization
       */
      virtual bool evaluate_continue(WidenEncodingState& state) = 0;
      virtual bool evaluate_continue(NarrowEncodingState& state) = 0;
    };
  } /* namespace internal */

  /**
   * EncodingErrorStrategy which replaces offending characters with valid ones
   */
  class ReplaceEncodingError: public internal::EncodingErrorStrategy
  {
    const wchar_t wide_replacement;
    const char narrow_replacement;

    /**
     * Checks the state result, inserting the replacement if necessary
     *
     * \tparam char_t a character type compatible with
     * encoding_state_t::replace_first
     * \tparam encoding_state_t a template instantiation of
     * internal::EncodingState
     * \param state the current encoding state including a valid result
     * \param replacement the character to insert on encoding errors
     */
    template<class char_t, class encoding_state_t>
    static bool replace_on_error(encoding_state_t& state, char_t replacement)
    {
      if(state.get_result() != internal::cvt_t::error)
    return false;
      state.replace_first(replacement);
      return true;
    }
  public:
    /**
     * \param wide_replacement a wide character valid in both locales used by
     * containing EncodingConverter
     * \param narrow_replacement a narrow character valid in the target encoding
     * used by the containing EncodingConverter
     */
    ReplaceEncodingError(wchar_t wide_replacement, char narrow_replacement):
      wide_replacement(wide_replacement), narrow_replacement(narrow_replacement)
    {}
    virtual ~ReplaceEncodingError() {}
    /**
     * \see internal::EncodingErrorStrategy::evaluate_continue
     */
    virtual bool evaluate_continue(internal::WidenEncodingState& state)
    {
      return replace_on_error(state, wide_replacement);
    }
    /**
     * \see internal::EncodingErrorStrategy::evaluate_continue
     */
    virtual bool evaluate_continue(internal::NarrowEncodingState& state)
    {
      return replace_on_error(state, narrow_replacement);
    }
  };
  /**
   * Default EncodingErrorStrategy used by EncodingConverter. Throws an
   * std::domain_error on encoding errors
   */
  class SignalEncodingError: public internal::EncodingErrorStrategy
  {
    /**
     * Produces an exception
     *
     * \param character the offending wide or narrow character cast to integer
     * \param loc the locale that could not handle the character
     * \throw std::bad_alloc if memory allocation fails
     * \throw std::domain_error in the normal case
     */
    static void encoding_error(int character, const std::locale& loc)
    {
      std::ostringstream err;
      err << "Invalid character 0x" << std::hex << character
      << " for locale " << loc.name();
    throw std::domain_error(err.str());
    }
    /**
     * Produces an exception if the encoding state contains an error
     *
     * \tparam encoding_state_t a template instantiation of
     * internal::EncodingState
     * \param state the current encoding state containing a result
     * \return false if there was no error
     * \throw std::domain_error on error
     * \throw std::bad_alloc if memory allocation fails
     */
    template<class encoding_state_t>
    static bool throw_on_error(encoding_state_t& state)
    {
      if(state.get_result() == internal::cvt_t::error)
    encoding_error(*(state.get_first_unconverted()), state.get_locale());
      return false;
    }
  public:
    virtual ~SignalEncodingError() {}
    /**
     * \see internal::EncodingErrorStrategy::evaluate_continue
     * \throw std::domain_error on error
     * \throw std::bad_alloc if memory allocation fails
     */
    virtual bool evaluate_continue(internal::WidenEncodingState& state)
    {
      return throw_on_error(state);
    }
    /**
     * \see internal::EncodingErrorStrategy::evaluate_continue
     * \throw std::domain_error on error
     * \throw std::bad_alloc if memory allocation fails
     */
    virtual bool evaluate_continue(internal::NarrowEncodingState& state)
    {
      return throw_on_error(state);
    }
  };
  /**
   * Converter between two encodings
   */
  class EncodingConverter
  {
    /**
     * locale determining the input encoding
     */
    const std::locale source_locale;
    /**
     * locale determining the output encoding
     */
    const std::locale target_locale;
    /**
     * internal buffer for widened characters
     *
     * std::wstring is not used because read/write access to the raw array
     * pointer is required
     */
    std::vector<wchar_t> wide;
    /**
     * internal buffer for characters in the target encoding
     *
     * std::string is not used because read/write access to the raw array
     * pointer is required
     */
    std::vector<char> narrow;
    /**
     * Strategy determining how encoding errors are handled. Never NULL
     */
    std::unique_ptr<internal::EncodingErrorStrategy> result_strategy;

    /**
     * Handles the actual conversion process
     *
     * Precondition: Sufficient memory is allocated
     * Postcondition: out is filled with valid characters and its size is
     * correctly set. In case of exceptions, out is in a valid but undefined
     * state
     *
     * \tparam source_char_t character type of the input. char or wchar_t
     * \tparam target_char_t character type of the output, wchar_t or char
     * \param locale determines the encoding used for the conversion
     * \param in pointer to a character array which is to be converted
     * \param in_end pointer after the end of in
     * \param out vector used for output. Memory needs to be pre-allocated
     * \throw std::bad_cast if locale does not have the cvt_t facet
     * \throw any std::exception if result_strategy wants to
     */
    template<class source_char_t, class target_char_t>
    void do_convert(const std::locale& locale, const source_char_t* in,
            const source_char_t* in_end,
            std::vector<target_char_t>& out)
    {
      typedef internal::EncodingState<const source_char_t, target_char_t>
    encoding_state_t;
      encoding_state_t state(locale, in, in_end, out.data(),
                 out.data() + out.size());
      do {
    state.encode();
      } while(state.get_result() != internal::cvt_t::ok
          && result_strategy->evaluate_continue(state));
      /* shrink out to the actual size */
      const std::size_t chars_n = std::distance(out.data(),
                        state.get_target_buf());
      out.resize(chars_n);
    }
    /**
     * Fills the wide member, updating its size
     *
     * On exceptions, wide remains valid but its content and size are undefined
     *
     * \param first input character in the source_locale encoding
     * \param last pointer after the last input character
     * \throw std::bad_alloc if memory allocation for wide fails
     * \throw std::bad_cast if source_locale does not have the cvt_t facet
     * \throw any std::exception if result_strategy wants to
     */
    void to_widechar(const char* first, const char* const last)
    {
      /* reserve enough memory for any conceivable interpretation of the input.
       * No encoding needs more characters in its wide representation than its
       * narrow multibyte or fixed size encoding requires bytes
       */
      const std::size_t bytes_n = std::distance(first, last);
      wide.resize(bytes_n);
      do_convert(source_locale, first, last, wide);
    }
    /**
     * Fills the narrow member with data from the wide member, updating its size
     *
     * Pre-condition: wide contains characters in the valid widened source
     * encoding
     * Post-condition: narrow contains only characters in the target encoding.
     * In case of exceptions, narrow remains valid but in undefined state
     *
     * \throw std::bad_alloc if memory allocation for narrow fails
     * \throw std::bad_cast if the target locale does not have the cvt_t facet
     * \throw any std::exception if result_strategy wants to
     */
    void to_multibyte()
    {
      /* reserve memory in narrow based on a worst-case estimate */
      const internal::cvt_t& converter =
    std::use_facet<internal::cvt_t>(target_locale);
      narrow.resize(wide.size() * converter.max_length());
      do_convert(target_locale, wide.data(), wide.data() + wide.size(), narrow);
    }
    EncodingConverter(const EncodingConverter& o);
    EncodingConverter& operator=(const EncodingConverter& o);
  public:
    /**
     * Initialized the EncodingConverter with the given locales and the
     * SignalEncodingError strategy
     *
     * \param locale_from a library-specific name of a locale, e.g. "en_US.UTF8"
     * \param locale_to a a library-specific name of a locale, e.g.
     * "en_US.ISO-8859-1"
     * \throw std::runtime_error if one of the locales is unsupported
     */
    EncodingConverter(const char* locale_from, const char* locale_to):
      source_locale(locale_from),
      target_locale(locale_to),
      result_strategy(new SignalEncodingError)
    {}
    /**
     * Changes the EncodingErrorStrategy
     *
     * \param strategy for handling encoding errors. Must not be NULL. Has to be
     * new-allocated. The EncodingConverter takes ownership
     */
    void set_error_behavior(internal::EncodingErrorStrategy* strategy)
    {
      result_strategy.reset(strategy);
    }
    /**
     * Transcodes the given characters from the source locale to the target
     * locale
     *
     * \param first pointer to a character array in the encoding used by the
     * source locale
     * \param last pointer behind the end of the character array specified by
     * first
     * \return a string in the encoding specified by the target locale
     * \throw std::bad_alloc if memory allocation fails
     * \throw std::bad_cast if one of the locales does not support encoding
     * conversion
     * \throw any std::exception on encoding errors depending on used
     * EncodingErrorStrategy
     */
    std::string operator()(const char* first, const char* last)
    {
      to_widechar(first, last);
      to_multibyte();
      return std::string(narrow.begin(), narrow.end());
    }
    /**
     * \see operator()(const char*, const char*)
     * \param c_str a '\0'-terminated string in the encoding specified by the
     * source locale
     */
    std::string operator()(const char* c_str)
    {
      return (*this)(c_str, c_str + std::strlen(c_str));
    }
    /**
     * \see operator()(const char*, const char*)
     * \param str a string in the encoding specified by the source locale
     */
    std::string operator()(const std::string& str)
    {
      return (*this)(str.data(), str.data() + str.size());
    }
  };
} /* namespace conv */

/**
 * Converts all command line parameters from UTF-8 to ISO-8859-1
 * Replaces characters that cannot be converted with '?'
 * Writes the output to stdout
 */
int main(int argc, char** argv)
{
  conv::EncodingConverter conv("de_DE.UTF-8", "en_US.ISO-8859-1");
  conv.set_error_behavior(new conv::ReplaceEncodingError(L'?', '?'));
  for(int i = 1; i < argc; ++i)
    std::cout << conv(argv[i]) << std::endl;
  return EXIT_SUCCESS;
}