More TextDecoder tests (BabylonJS#141)

`std::wstring_convert` is too strict and doesn't accept strings with
invalid characters.
Other impl (Chakra, V8) and browsers accept such strings.

Author: CedricGuillemet

Core/Node-API/Source/js_native_api_javascriptcore.cc

@@ -7,8 +7,7 @@
 #include <stdexcept>
 #include <string>
 #include <vector>
-#include <locale>
-#include <codecvt>
+
 
 struct napi_callback_info__ {
   napi_value newTarget;
@@ -103,13 +102,68 @@ namespace {
     }
 
    private:
+    // Decode UTF-8 to UTF-16, replacing invalid sequences with U+FFFD.
+    static std::u16string Utf8ToUtf16(const char* str, size_t len) {
+      std::u16string result;
+      result.reserve(len);
+      const auto* s = reinterpret_cast<const unsigned char*>(str);
+      const auto* end = s + len;
+      while (s < end) {
+        uint32_t cp;
+        int trail;
+        unsigned char lead = *s++;
+        if (lead < 0x80) {
+          cp = lead; trail = 0;
+        } else if ((lead >> 5) == 0x6) {
+          cp = lead & 0x1F; trail = 1;
+        } else if ((lead >> 4) == 0xE) {
+          cp = lead & 0x0F; trail = 2;
+        } else if ((lead >> 3) == 0x1E) {
+          cp = lead & 0x07; trail = 3;
+        } else {
+          result.push_back(0xFFFD);
+          continue;
+        }
+        if (s + trail > end) {
+          result.push_back(0xFFFD);
+          break;
+        }
+        bool valid = true;
+        for (int i = 0; i < trail; ++i) {
+          if ((s[i] & 0xC0) != 0x80) { valid = false; break; }
+          cp = (cp << 6) | (s[i] & 0x3F);
+        }
+        if (!valid) {
+          result.push_back(0xFFFD);
+          continue;
+        }
+        s += trail;
+        // Reject overlong encodings, surrogates, and out-of-range values.
+        if ((trail == 1 && cp < 0x80) ||
+            (trail == 2 && cp < 0x800) ||
+            (trail == 3 && cp < 0x10000) ||
+            (cp >= 0xD800 && cp <= 0xDFFF) ||
+            cp > 0x10FFFF) {
+          result.push_back(0xFFFD);
+          continue;
+        }
+        if (cp <= 0xFFFF) {
+          result.push_back(static_cast<char16_t>(cp));
+        } else {
+          cp -= 0x10000;
+          result.push_back(static_cast<char16_t>(0xD800 + (cp >> 10)));
+          result.push_back(static_cast<char16_t>(0xDC00 + (cp & 0x3FF)));
+        }
+      }
+      return result;
+    }
+
     static JSStringRef CreateUTF8(const char* string, size_t length) {
       if (length == NAPI_AUTO_LENGTH) {
         return JSStringCreateWithUTF8CString(string);
       }
 
-      std::u16string u16str{std::wstring_convert<
-        std::codecvt_utf8_utf16<char16_t>, char16_t>{}.from_bytes(string, string + length)};
+      std::u16string u16str{Utf8ToUtf16(string, length)};
       return JSStringCreateWithCharacters(reinterpret_cast<JSChar*>(u16str.data()), u16str.size());
     }
 

Tests/UnitTests/Android/app/build.gradle

@@ -88,6 +88,12 @@ task copyScripts {
             include "*.js"
             into 'src/main/assets/Scripts'
         }
+        copy
+        {
+            from '../../Assets'
+            include "*.*"
+            into 'src/main/assets/Assets'
+        }
     }
 }
 

Tests/UnitTests/Assets/Halo_Believe.ply

@@ -5,6 +5,8 @@ set(SCRIPTS
 set(TYPE_SCRIPTS
     "Scripts/tests.ts")
 
+file(GLOB ASSETS "${CMAKE_CURRENT_SOURCE_DIR}/Assets/*")
+
 set(SOURCES
     "Shared/Shared.cpp"
     "Shared/Shared.h")
@@ -25,6 +27,10 @@ if(APPLE)
             ${SCRIPTS}
             ${TYPE_SCRIPTS}
             PROPERTIES MACOSX_PACKAGE_LOCATION "Scripts")
+
+        set_source_files_properties(
+            ${ASSETS}
+            PROPERTIES MACOSX_PACKAGE_LOCATION "Assets")
     else()
         set(SOURCES ${SOURCES}
             macOS/App.mm)
@@ -37,7 +43,7 @@ elseif(UNIX AND NOT ANDROID)
         Linux/App.cpp)
 endif()
 
-add_executable(UnitTests ${SOURCES} ${SCRIPTS} ${TYPE_SCRIPTS})
+add_executable(UnitTests ${SOURCES} ${SCRIPTS} ${TYPE_SCRIPTS} ${ASSETS})
 target_compile_definitions(UnitTests PRIVATE JSRUNTIMEHOST_PLATFORM="${JSRUNTIMEHOST_PLATFORM}")
 
 target_link_libraries(UnitTests
@@ -87,9 +93,17 @@ else()
         COMMAND ${CMAKE_COMMAND} -E create_symlink "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR}/Scripts/symlink_target.js" "${CMAKE_CFG_INTDIR}/Scripts/symlink_1.js")
     add_custom_command(TARGET UnitTests POST_BUILD
         COMMAND ${CMAKE_COMMAND} -E create_symlink "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR}/Scripts/symlink_1.js" "${CMAKE_CFG_INTDIR}/Scripts/symlink_2.js")
+
+    foreach(ASSET ${ASSETS})
+        get_filename_component(ASSET_NAME "${ASSET}" NAME)
+        add_custom_command(TARGET UnitTests POST_BUILD
+            COMMAND "${CMAKE_COMMAND}" -E copy "${ASSET}" "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_CFG_INTDIR}/Assets/${ASSET_NAME}"
+            COMMENT "Copying ${ASSET_NAME}")
+    endforeach()
 endif()
 
 set_property(TARGET UnitTests PROPERTY FOLDER Tests)
 
 source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR} FILES ${SOURCES} ${SCRIPTS})
 source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR}/Scripts PREFIX scripts FILES ${TYPE_SCRIPTS})
+source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR}/Assets PREFIX assets FILES ${ASSETS})

Tests/UnitTests/CMakeLists.txt

@@ -196,6 +196,21 @@ describe("XMLHTTPRequest", function () {
         var response = new Uint8Array(xhr.response);
         expect(response).to.eql(expected);
     });
+
+    it("should load a PLY file and parse vertex count from header using TextDecoder", async function () {
+        this.timeout(30000);
+        const xhr = await createRequest("GET", "app:///Assets/Halo_Believe.ply", undefined, "arraybuffer");
+        expect(xhr.status).to.equal(200);
+
+        const ubuf = new Uint8Array(xhr.response);
+        const header = new TextDecoder().decode(ubuf.slice(0, 1024 * 10));
+        const headerEnd = "end_header\n";
+        const headerEndIndex = header.indexOf(headerEnd);
+        expect(headerEndIndex).to.be.greaterThan(0);
+
+        const vertexCount = parseInt(/element vertex (\d+)\n/.exec(header)![1]);
+        expect(vertexCount).to.equal(18713);
+    });
 });
 
 describe("setTimeout", function () {
@@ -1268,6 +1283,14 @@ describe("TextDecoder", function () {
         const result = decoder.decode(sub);
         expect(result).to.equal("Hi");
     });
+
+    it("should decode a Uint8Array containing a null byte", function () {
+        const decoder = new TextDecoder();
+        const encoded = new Uint8Array([72, 0, 105]); // "H\0i"
+        const result = decoder.decode(encoded);
+        expect(result).to.equal("H\0i");
+        expect(result.length).to.equal(3);
+    });
 });
 
 function runTests() {