Strutil utf conversion improvements (#3553)

* Add `Strutil::utf16_to_utf8(const std::u16string& utf16str)`.
  This is subtly different from our existing utf16_to_utf8() that
  takes a std::wstring, because wchar_t is not the same on all
  platforms.  If it really and truly is utf16-in-char16_t, you need
  this and can't use the utf-in-wchar_t which may be 32 bit chars.

* Add `Strutil::utf8_to_utf16wstring()` (identical to the old
  utr8_to_utf16) to make it more self-documenting as returning a
  utf16-encoded wstring and NOT a u16string. Note that wchar_t is not
  necessarily char16_t on all platforms (it is on Windows, but it's 32
  bits on Linux).

* Deprecate the ambiguously named `utf8_to_utf16()` for OIIO >= 2.5 and
  remove it for OIIO >= 3.0.

src/include/OpenImageIO/strutil.h

@@ -834,15 +834,29 @@ struct OIIO_UTIL_API StringILess {
 
 
 /// Conversion of normal char-based strings (presumed to be UTF-8 encoding)
-/// to wide char string, wstring.
+/// to a UTF-16 encoded wide char string, wstring.
+std::wstring OIIO_UTIL_API utf8_to_utf16wstring (string_view utf8str) noexcept;
+
+#if OPENIMAGEIO_VERSION < 30000
+/// Old name for utf8_to_utf16wstring. Will be deprecated for OIIO 2.5+ and
+/// removed for OIIO 3.0. Use utf8_to_utf16wstring which is more clear that
+/// this particular conversion from utf8 to utf16 returns a std::wstring and
+/// not a std::u16string.
+#if OPENIMAGEIO_VERSION >= 20500
+OIIO_DEPRECATED("Use utf8_to_utf16wstring instead")
+#endif
 std::wstring OIIO_UTIL_API utf8_to_utf16 (string_view utf8str) noexcept;
+#endif
 
 /// Conversion from wstring UTF-16 to a UTF-8 std::string.  This is the
 /// standard way to convert from Windows wide character strings used for
 /// filenames into the UTF-8 strings OIIO expects for filenames when passed to
 /// functions like ImageInput::open().
 std::string OIIO_UTIL_API utf16_to_utf8(const std::wstring& utf16str) noexcept;
 
+/// Conversion from UTF-16 std::u16string to a UTF-8 std::string.
+std::string OIIO_UTIL_API utf16_to_utf8(const std::u16string& utf16str) noexcept;
+
 
 
 /// Copy at most size characters (including terminating 0 character) from

src/libutil/filesystem.cpp

@@ -56,7 +56,7 @@ inline filesystem::path
 u8path(string_view name)
 {
 #ifdef _WIN32
-    return filesystem::path(Strutil::utf8_to_utf16(name));
+    return filesystem::path(Strutil::utf8_to_utf16wstring(name));
 #else
     return filesystem::path(name.begin(), name.end());
 #endif
@@ -66,7 +66,7 @@ inline filesystem::path
 u8path(const std::string& name)
 {
 #ifdef _WIN32
-    return filesystem::path(Strutil::utf8_to_utf16(name));
+    return filesystem::path(Strutil::utf8_to_utf16wstring(name));
 #else
     return filesystem::path(name);
 #endif
@@ -456,7 +456,7 @@ Filesystem::unique_path(string_view model)
     // to convert char* to wchar_t* because they do not know the encoding
     // See boost/filesystem/path.hpp
     // The only correct way to do this is to do the conversion ourselves
-    std::wstring modelStr    = Strutil::utf8_to_utf16(model);
+    std::wstring modelStr    = Strutil::utf8_to_utf16wstring(model);
 #    else
     std::string modelStr = model.str();
 #    endif
@@ -495,9 +495,8 @@ Filesystem::fopen(string_view path, string_view mode)
 {
 #ifdef _WIN32
     // on Windows fopen does not accept UTF-8 paths, so we convert to wide char
-    std::wstring wpath = Strutil::utf8_to_utf16(path);
-    std::wstring wmode = Strutil::utf8_to_utf16(mode);
-
+    std::wstring wpath = Strutil::utf8_to_utf16wstring(path);
+    std::wstring wmode = Strutil::utf8_to_utf16wstring(mode);
     return ::_wfopen(wpath.c_str(), wmode.c_str());
 #else
     // on Unix platforms passing in UTF-8 works
@@ -538,7 +537,7 @@ Filesystem::open(OIIO::ifstream& stream, string_view path,
 #ifdef _WIN32
     // Windows std::ifstream accepts non-standard wchar_t*
     // On MingW, we use our own OIIO::ifstream
-    std::wstring wpath = Strutil::utf8_to_utf16(path);
+    std::wstring wpath = Strutil::utf8_to_utf16wstring(path);
     stream.open(wpath.c_str(), mode);
     stream.seekg(0, std::ios_base::beg);  // force seek, otherwise broken
 #else
@@ -555,7 +554,7 @@ Filesystem::open(OIIO::ofstream& stream, string_view path,
 #ifdef _WIN32
     // Windows std::ofstream accepts non-standard wchar_t*
     // On MingW, we use our own OIIO::ofstream
-    std::wstring wpath = Strutil::utf8_to_utf16(path);
+    std::wstring wpath = Strutil::utf8_to_utf16wstring(path);
     stream.open(wpath.c_str(), mode);
 #else
     stream.open(path, mode);
@@ -570,7 +569,7 @@ Filesystem::open(string_view path, int flags)
 #ifdef _WIN32
     // on Windows _open does not accept UTF-8 paths, so we convert to wide
     // char and use _wopen.
-    std::wstring wpath = Strutil::utf8_to_utf16(path);
+    std::wstring wpath = Strutil::utf8_to_utf16wstring(path);
     return ::_wopen(wpath.c_str(), flags);
 #else
     // on Unix platforms passing in UTF-8 works

src/libutil/plugin.cpp

@@ -48,7 +48,7 @@ Plugin::plugin_extension(void)
 Handle
 dlopen(const char* plugin_filename, int)
 {
-    std::wstring w = Strutil::utf8_to_utf16(plugin_filename);
+    std::wstring w = Strutil::utf8_to_utf16wstring(plugin_filename);
     return LoadLibraryW(w.c_str());
 }
 

src/libutil/strutil.cpp

@@ -870,7 +870,7 @@ Strutil::replace(string_view str, string_view pattern, string_view replacement,
 // appropriate for error messages.
 
 std::wstring
-Strutil::utf8_to_utf16(string_view str) noexcept
+Strutil::utf8_to_utf16wstring(string_view str) noexcept
 {
     try {
         std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>, wchar_t> conv;
@@ -882,6 +882,17 @@ Strutil::utf8_to_utf16(string_view str) noexcept
 
 
 
+#if OPENIMAGEIO_VERSION < 30000
+// DEPRECATED(2.5) and slated for removal in 3.0.
+std::wstring
+Strutil::utf8_to_utf16(string_view str) noexcept
+{
+    return utf8_to_utf16wstring(str);
+}
+#endif
+
+
+
 std::string
 Strutil::utf16_to_utf8(const std::wstring& str) noexcept
 {
@@ -895,6 +906,19 @@ Strutil::utf16_to_utf8(const std::wstring& str) noexcept
 
 
 
+std::string
+Strutil::utf16_to_utf8(const std::u16string& str) noexcept
+{
+    try {
+        std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> conv;
+        return conv.to_bytes(str);
+    } catch (const std::exception&) {
+        return std::string();
+    }
+}
+
+
+
 char*
 Strutil::safe_strcpy(char* dst, string_view src, size_t size) noexcept
 {

src/raw.imageio/rawinput.cpp

@@ -411,7 +411,8 @@ RawInput::open_raw(bool unpack, const std::string& name,
 
 #ifdef _WIN32
     // Convert to wide chars, just on Windows.
-    int ret = m_processor->open_file(Strutil::utf8_to_utf16(name).c_str());
+    int ret = m_processor->open_file(
+        Strutil::utf8_to_utf16wstring(name).c_str());
 #else
     int ret = m_processor->open_file(name.c_str());
 #endif

src/tiff.imageio/tiffinput.cpp

@@ -742,7 +742,7 @@ TIFFInput::seek_subimage(int subimage, int miplevel)
                                    reader_unmapproc);
         } else {
 #ifdef _WIN32
-            std::wstring wfilename = Strutil::utf8_to_utf16(m_filename);
+            std::wstring wfilename = Strutil::utf8_to_utf16wstring(m_filename);
             m_tif                  = TIFFOpenW(wfilename.c_str(), "rm");
 #else
             m_tif = TIFFOpen(m_filename.c_str(), "rm");

src/tiff.imageio/tiffoutput.cpp

@@ -448,7 +448,8 @@ TIFFOutput::open(const std::string& name, const ImageSpec& userspec,
                                writer_unmapproc);
     } else {
 #ifdef _WIN32
-        m_tif = TIFFOpenW(Strutil::utf8_to_utf16(name).c_str(), openmode);
+        m_tif = TIFFOpenW(Strutil::utf8_to_utf16wstring(name).c_str(),
+                          openmode);
 #else
         m_tif = TIFFOpen(name.c_str(), openmode);
 #endif

src/zfile.imageio/zfile.cpp

@@ -41,7 +41,7 @@ gzFile
 open_gz(const std::string& filename, const char* mode)
 {
 #ifdef _WIN32
-    std::wstring wpath = Strutil::utf8_to_utf16(filename);
+    std::wstring wpath = Strutil::utf8_to_utf16wstring(filename);
     gzFile gz          = gzopen_w(wpath.c_str(), mode);
 #else
     gzFile gz = gzopen(filename.c_str(), mode);