Introduce fsyspath subclass of std::filesystem::path.

Our std::strings are UTF-8 encoded, so conversion from std::string to
std::filesystem::path must use UTF-8 decoding. The native Windows
std::filesystem::path constructor and assignment operator accepting
std::string use "native narrow encoding," which mangles path strings
containing UTF-8 encoded non-ASCII characters.

fsyspath's std::string constructor and assignment operator explicitly engage
std::filesystem::u8path() to handle encoding. u8path() is deprecated in C++20,
but once we adapt fsyspath's conversion to C++20 conventions, consuming code
need not be modified.

indra/llcommon/CMakeLists.txt

@@ -127,6 +127,7 @@ set(llcommon_HEADER_FILES
     commoncontrol.h
     ctype_workaround.h
     fix_macros.h
+    fsyspath.h
     function_types.h
     indra_constants.h
     lazyeventapi.h

indra/llcommon/fsyspath.h

@@ -0,0 +1,74 @@
+/**
+ * @file   fsyspath.h
+ * @author Nat Goodspeed
+ * @date   2024-04-03
+ * @brief  Adapt our UTF-8 std::strings for std::filesystem::path
+ * 
+ * $LicenseInfo:firstyear=2024&license=viewerlgpl$
+ * Copyright (c) 2024, Linden Research, Inc.
+ * $/LicenseInfo$
+ */
+
+#if ! defined(LL_FSYSPATH_H)
+#define LL_FSYSPATH_H
+
+#include <filesystem>
+
+// While std::filesystem::path can be directly constructed from std::string on
+// both Posix and Windows, that's not what we want on Windows. Per
+// https://en.cppreference.com/w/cpp/filesystem/path/path:
+
+// ... the method of conversion to the native character set depends on the
+// character type used by source.
+// 
+// * If the source character type is char, the encoding of the source is
+//   assumed to be the native narrow encoding (so no conversion takes place on
+//   POSIX systems).
+// * If the source character type is char8_t, conversion from UTF-8 to native
+//   filesystem encoding is used. (since C++20)
+// * If the source character type is wchar_t, the input is assumed to be the
+//   native wide encoding (so no conversion takes places on Windows).
+
+// The trouble is that on Windows, from std::string ("source character type is
+// char"), the "native narrow encoding" isn't UTF-8, so file paths containing
+// non-ASCII characters get mangled.
+//
+// Once we're building with C++20, we could pass a UTF-8 std::string through a
+// vector<char8_t> to engage std::filesystem::path's own UTF-8 conversion. But
+// sigh, as of 2024-04-03 we're not yet there.
+//
+// Anyway, encapsulating the important UTF-8 conversions in our own subclass
+// allows us to migrate forward to C++20 conventions without changing
+// referencing code.
+
+class fsyspath: public std::filesystem::path
+{
+    using super = std::filesystem::path;
+
+public:
+    // default
+    fsyspath() {}
+    // construct from UTF-8 encoded std::string
+    fsyspath(const std::string& path): super(std::filesystem::u8path(path)) {}
+    // construct from UTF-8 encoded const char*
+    fsyspath(const char* path): super(std::filesystem::u8path(path)) {}
+    // construct from existing path
+    fsyspath(const super& path): super(path) {}
+
+    fsyspath& operator=(const super& p) { super::operator=(p); return *this; }
+    fsyspath& operator=(const std::string& p)
+    {
+        super::operator=(std::filesystem::u8path(p));
+        return *this;
+    }
+    fsyspath& operator=(const char* p)
+    {
+        super::operator=(std::filesystem::u8path(p));
+        return *this;
+    }
+
+    // shadow base-class string() method with UTF-8 aware method
+    std::string string() const { return super::u8string(); }
+};
+
+#endif /* ! defined(LL_FSYSPATH_H) */

indra/llcommon/lua_function.cpp

@@ -17,13 +17,13 @@
 // std headers
 #include <algorithm>
 #include <exception>
-#include <filesystem>
 #include <iomanip>                  // std::quoted
 #include <map>
 #include <memory>                   // std::unique_ptr
 #include <typeinfo>
 // external library headers
 // other Linden headers
+#include "fsyspath.h"
 #include "hexdump.h"
 #include "lleventcoro.h"
 #include "llsd.h"
@@ -68,7 +68,7 @@ int lluau::loadstring(lua_State *L, const std::string &desc, const std::string &
     return luau_load(L, desc.data(), bytecode.get(), bytecodeSize, 0);
 }
 
-std::filesystem::path lluau::source_path(lua_State* L)
+fsyspath lluau::source_path(lua_State* L)
 {
     //Luau lua_Debug and lua_getinfo() are different compared to default Lua:
     //see https://github.com/luau-lang/luau/blob/80928acb92d1e4b6db16bada6d21b1fb6fa66265/VM/include/lua.h
@@ -577,7 +577,7 @@ std::pair<int, LLSD> LuaState::expr(const std::string& desc, const std::string&
         // the next call to lua_next."
         // https://www.lua.org/manual/5.1/manual.html#lua_next
         if (lua_type(mState, -2) == LUA_TSTRING &&
-            std::filesystem::path(lua_tostdstring(mState, -2)).stem() == "fiber")
+            fsyspath(lua_tostdstring(mState, -2)).stem() == "fiber")
         {
             found = true;
             break;

indra/llcommon/lua_function.h

@@ -16,9 +16,9 @@
 #include "luau/lua.h"
 #include "luau/luaconf.h"
 #include "luau/lualib.h"
+#include "fsyspath.h"
 #include "stringize.h"
 #include <exception>                // std::uncaught_exceptions()
-#include <filesystem>
 #include <memory>                   // std::shared_ptr
 #include <utility>                  // std::pair
 
@@ -51,7 +51,7 @@ namespace lluau
     int dostring(lua_State* L, const std::string& desc, const std::string& text);
     int loadstring(lua_State* L, const std::string& desc, const std::string& text);
 
-    std::filesystem::path source_path(lua_State* L);
+    fsyspath source_path(lua_State* L);
 } // namespace lluau
 
 std::string lua_tostdstring(lua_State* L, int index);

indra/llui/llluafloater.cpp

@@ -26,7 +26,7 @@
 
 #include "llluafloater.h"
 
-#include <filesystem>
+#include "fsyspath.h"
 #include "llevents.h"
 
 #include "llcheckboxctrl.h"
@@ -271,12 +271,12 @@ void LLLuaFloater::postEvent(LLSD data, const std::string &event_name)
 
 void LLLuaFloater::showLuaFloater(const LLSD &data)
 {
-    std::filesystem::path fs_path(data["xml_path"].asString());
-    std::string path = fs_path.lexically_normal().string();
+    fsyspath fs_path(data["xml_path"].asString());
+    std::string path = fs_path.lexically_normal().u8string();
     if (!fs_path.is_absolute()) 
     {
         std::string lib_path = gDirUtilp->getExpandedFilename(LL_PATH_SCRIPTS, "lua");
-        path = (std::filesystem::path(lib_path) / path).u8string();
+        path = (fsyspath(lib_path) / path).u8string();
     }
 
     LLLuaFloater *floater = new LLLuaFloater(data);

indra/newview/llluamanager.cpp

@@ -28,6 +28,7 @@
 #include "llviewerprecompiledheaders.h"
 #include "llluamanager.h"
 
+#include "fsyspath.h"
 #include "llcoros.h"
 #include "llerror.h"
 #include "lleventcoro.h"
@@ -37,7 +38,6 @@
 #include "stringize.h"
 
 #include <boost/algorithm/string/replace.hpp>
-#include <filesystem>
 
 #include "luau/luacode.h"
 #include "luau/lua.h"
@@ -314,7 +314,7 @@ void LLRequireResolver::resolveRequire(lua_State *L, std::string path)
 }
 
 LLRequireResolver::LLRequireResolver(lua_State *L, const std::string& path) :
-    mPathToResolve(std::filesystem::path(path).lexically_normal()),
+    mPathToResolve(fsyspath(path).lexically_normal()),
     L(L)
 {
     mSourceDir = lluau::source_path(L).parent_path();
@@ -377,12 +377,12 @@ void LLRequireResolver::findModule()
         fail();
     }
 
-    std::vector<std::filesystem::path> lib_paths
+    std::vector<fsyspath> lib_paths
     {
         gDirUtilp->getExpandedFilename(LL_PATH_SCRIPTS, "lua"),
 #ifdef LL_TEST
         // Build-time tests don't have the app bundle - use source tree.
-        std::filesystem::path(__FILE__).parent_path() / "scripts" / "lua",
+        fsyspath(__FILE__).parent_path() / "scripts" / "lua",
 #endif
     };
 

indra/newview/llluamanager.h

@@ -27,9 +27,9 @@
 #ifndef LL_LLLUAMANAGER_H
 #define LL_LLLUAMANAGER_H
 
+#include "fsyspath.h"
 #include "llcoros.h"
 #include "llsd.h"
-#include <filesystem>
 #include <functional>
 #include <string>
 #include <utility>                  // std::pair
@@ -89,8 +89,8 @@ class LLRequireResolver
     static void resolveRequire(lua_State *L, std::string path);
 
  private:
-    std::filesystem::path mPathToResolve;
-    std::filesystem::path mSourceDir;
+    fsyspath mPathToResolve;
+    fsyspath mSourceDir;
 
     LLRequireResolver(lua_State *L, const std::string& path);