fs: support encoding option in glob

Add an `encoding` option to `fs.glob()`, `fs.globSync()`, and
`fs.promises.glob()`. The glob walker already operates on raw path
bytes; encode results at the native-to-JavaScript boundary instead so
that filenames which are not valid UTF-8 can be returned without loss
when `encoding` is `'buffer'`.

With `withFileTypes`, the `name` and `parentPath` properties of each
Dirent, as well as the values passed to `exclude` callbacks, use the
requested encoding. Filesystem operations, including the lstat used
for path callbacks, keep using the original paths. The default
encoding remains `'utf8'`.

Refs: #59202
Signed-off-by: XadillaX <i@2333.moe>
Assisted-by: Trae

Author: XadillaX

doc/api/fs.md

@@ -1461,6 +1461,9 @@ changes:
 * `pattern` {string|string\[]}
 * `options` {Object}
   * `cwd` {string|URL} current working directory. **Default:** `process.cwd()`
+  * `encoding` {string} Encoding for returned paths and paths passed to an
+    `exclude` function. Use `'buffer'` to preserve filename bytes.
+    **Default:** `'utf8'`.
   * `exclude` {Function|string\[]} Function to filter out files/directories or a
     list of [glob patterns][] to be excluded. If a function is provided, return
     `true` to exclude the item, `false` to include it. **Default:** `undefined`.
@@ -1478,6 +1481,11 @@ changes:
 
 See [Glob patterns][] for the syntax `pattern` accepts.
 
+When `encoding` is `'buffer'`, paths are returned as {Buffer} objects.
+With `withFileTypes: true`, the `name` and `parentPath` properties of each
+{fs.Dirent} use the specified encoding, including those passed to `exclude`.
+The encoding does not change how patterns are matched.
+
 When `followSymlinks` is enabled, detected symbolic link cycles are not
 traversed recursively.
 
@@ -3744,6 +3752,9 @@ changes:
 
 * `options` {Object}
   * `cwd` {string|URL} current working directory. **Default:** `process.cwd()`
+  * `encoding` {string} Encoding for returned paths and paths passed to an
+    `exclude` function. Use `'buffer'` to preserve filename bytes.
+    **Default:** `'utf8'`.
   * `exclude` {Function|string\[]} Function to filter out files/directories or a
     list of [glob patterns][] to be excluded. If a function is provided, return
     `true` to exclude the item, `false` to include it. **Default:** `undefined`.
@@ -3761,6 +3772,11 @@ changes:
 
 See [Glob patterns][] for the syntax `pattern` accepts.
 
+When `encoding` is `'buffer'`, paths are returned as {Buffer} objects.
+With `withFileTypes: true`, the `name` and `parentPath` properties of each
+{fs.Dirent} use the specified encoding, including those passed to `exclude`.
+The encoding does not change how patterns are matched.
+
 When `followSymlinks` is enabled, detected symbolic link cycles are not
 traversed recursively.
 
@@ -6460,6 +6476,9 @@ changes:
 * `pattern` {string|string\[]}
 * `options` {Object}
   * `cwd` {string|URL} current working directory. **Default:** `process.cwd()`
+  * `encoding` {string} Encoding for returned paths and paths passed to an
+    `exclude` function. Use `'buffer'` to preserve filename bytes.
+    **Default:** `'utf8'`.
   * `exclude` {Function|string\[]} Function to filter out files/directories or a
     list of [glob patterns][] to be excluded. If a function is provided, return
     `true` to exclude the item, `false` to include it. **Default:** `undefined`.
@@ -6469,10 +6488,16 @@ changes:
     The `cwd` directory has a depth of `0`. **Default:** `Infinity`.
   * `withFileTypes` {boolean} `true` if the glob should return paths as Dirents,
     `false` otherwise. **Default:** `false`.
-* Returns: {string\[]} paths of files that match the pattern.
+* Returns: {string\[]|Buffer\[]|fs.Dirent\[]} paths or directory entries that
+  match the pattern.
 
 See [Glob patterns][] for the syntax `pattern` accepts.
 
+When `encoding` is `'buffer'`, paths are returned as {Buffer} objects.
+With `withFileTypes: true`, the `name` and `parentPath` properties of each
+{fs.Dirent} use the specified encoding, including those passed to `exclude`.
+The encoding does not change how patterns are matched.
+
 When `followSymlinks` is enabled, detected symbolic link cycles are not
 traversed recursively.
 
@@ -7731,8 +7756,8 @@ added: v10.10.0
 * Type: {string|Buffer}
 
 The file name that this {fs.Dirent} object refers to. The type of this
-value is determined by the `options.encoding` passed to [`fs.readdir()`][] or
-[`fs.readdirSync()`][].
+value is determined by the `options.encoding` passed to the directory-reading
+or glob function that created it.
 
 #### `dirent.parentPath`
 
@@ -7749,9 +7774,10 @@ changes:
     description: Marking the API stable.
 -->
 
-* Type: {string}
+* Type: {string|Buffer}
 
 The path to the parent directory of the file this {fs.Dirent} object refers to.
+For glob results, this value uses the specified `options.encoding`.
 
 ### Class: `fs.FSWatcher`
 

lib/internal/fs/glob.js

@@ -7,7 +7,7 @@ const {
 } = primordials;
 
 const { lstatSync } = require('fs');
-const { basename, dirname, isAbsolute, join, resolve } = require('path');
+const { resolve } = require('path');
 
 const {
   kEmptyObject,
@@ -20,7 +20,7 @@ const {
   validateString,
   validateStringArray,
 } = require('internal/validators');
-const { Dirent, DirentFromStats } = require('internal/fs/utils');
+const { assertEncoding, Dirent, DirentFromStats } = require('internal/fs/utils');
 const {
   codes: {
     ERR_INVALID_ARG_TYPE,
@@ -41,17 +41,19 @@ const {
 } = globBinding;
 
 /**
- * @param {string} path
+ * @param {Buffer} path
+ * @param {string|Buffer} name
+ * @param {string|Buffer} parentPath
  * @returns {DirentFromStats|null}
  */
-function getDirentSync(path) {
+function getDirentSync(path, name, parentPath) {
   let stat;
   try {
     stat = lstatSync(path);
   } catch {
     return null;
   }
-  return new DirentFromStats(basename(path), stat, dirname(path));
+  return new DirentFromStats(name, stat, parentPath);
 }
 
 /**
@@ -75,12 +77,17 @@ class Glob {
   #excludePatterns;
   #excludeFn;
   #withFileTypes;
+  #encoding;
   #followSymlinks = false;
   #maxDepth = Infinity;
 
   constructor(pattern, options = kEmptyObject) {
     validateObject(options, 'options');
-    const { exclude, cwd, followSymlinks, maxDepth, withFileTypes } = options;
+    const { exclude, cwd, encoding, followSymlinks, maxDepth, withFileTypes } = options;
+    if (encoding !== 'buffer') {
+      assertEncoding(encoding);
+    }
+    this.#encoding = encoding || 'utf8';
     this.#root = toPathIfFileURL(cwd) ?? '.';
     validateString(this.#root, 'options.cwd');
     if (followSymlinks != null) {
@@ -126,30 +133,30 @@ class Glob {
   globSync() {
     const result = globBinding.globSync(
       this.#root, this.#patterns, this.#excludePatterns, this.#flags(),
-      this.#excludeAdapter(), this.#maxDepth);
+      this.#excludeAdapter(), this.#maxDepth, this.#encoding);
     if (!this.#withFileTypes) {
       return result;
     }
-    const { 0: paths, 1: types } = result;
+    const { 0: names, 1: types, 2: parents } = result;
     const dirents = [];
-    for (let i = 0; i < paths.length; i++) {
-      ArrayPrototypePush(dirents, this.#toDirent(paths[i], types[i]));
+    for (let i = 0; i < names.length; i++) {
+      ArrayPrototypePush(dirents, new Dirent(names[i], types[i], parents[i]));
     }
     return dirents;
   }
 
   async globAll() {
     const handle = globBinding.globStart(
       this.#root, this.#patterns, this.#excludePatterns, this.#flags(),
-      this.#excludeAdapter(), this.#maxDepth);
+      this.#excludeAdapter(), this.#maxDepth, this.#encoding);
     try {
-      const { 0: paths, 1: types } = await handle.all();
+      const { 0: paths, 1: types, 3: parents } = await handle.all();
       if (!this.#withFileTypes) {
         return paths;
       }
       const dirents = [];
       for (let i = 0; i < paths.length; i++) {
-        ArrayPrototypePush(dirents, this.#toDirent(paths[i], types[i]));
+        ArrayPrototypePush(dirents, new Dirent(paths[i], types[i], parents[i]));
       }
       return dirents;
     } finally {
@@ -161,14 +168,14 @@ class Glob {
   async * glob() {
     const handle = globBinding.globStart(
       this.#root, this.#patterns, this.#excludePatterns, this.#flags(),
-      this.#excludeAdapter(), this.#maxDepth);
+      this.#excludeAdapter(), this.#maxDepth, this.#encoding);
     try {
       let done = false;
       while (!done) {
-        const { 0: paths, 1: types, 2: batchDone } = await handle.next();
+        const { 0: paths, 1: types, 2: batchDone, 3: parents } = await handle.next();
         done = batchDone;
         for (let i = 0; i < paths.length; i++) {
-          yield this.#withFileTypes ? this.#toDirent(paths[i], types[i]) : paths[i];
+          yield this.#withFileTypes ? new Dirent(paths[i], types[i], parents[i]) : paths[i];
         }
       }
     } finally {
@@ -191,19 +198,14 @@ class Glob {
     if (!this.#withFileTypes) {
       return (entry, value) => exclude(value);
     }
-    return (entry, value, parentPath, type) => {
+    return (entry, value, parentPath, type, rawPath) => {
       if (entry) {
         return exclude(new Dirent(value, type, parentPath));
       }
-      const dirent = getDirentSync(value);
+      const dirent = getDirentSync(rawPath, value, parentPath);
       return dirent === null ? false : exclude(dirent);
     };
   }
-
-  #toDirent(path, type) {
-    const full = isAbsolute(path) ? path : join(this.#root, path);
-    return new Dirent(basename(full), type, dirname(full));
-  }
 }
 
 /**

src/glob/glob_walker.cc

@@ -1782,6 +1782,73 @@ void ScanPool::RunOne(Mutex::ScopedLock& lock) {
 
 }  // namespace
 
+DirentPath GetDirentPath(const std::string& cwd, const std::string& path) {
+  std::string full = path;
+  const bool joined = !cwd.empty() && !IsAbsolute(path);
+  if (joined) {
+    full = cwd + kSep + path;
+    if constexpr (kWindows) {
+      // Like path.join(), only retain a UNC prefix supplied by cwd itself.
+      const bool unc = cwd.size() > 2 && IsPathSeparator(cwd[0]) &&
+                       IsPathSeparator(cwd[1]) && !IsPathSeparator(cwd[2]);
+      if (!unc) {
+        size_t leading = 0;
+        while (leading < full.size() && IsPathSeparator(full[leading])) {
+          leading++;
+        }
+        if (leading > 1) full.erase(0, leading - 1);
+      }
+    }
+    full = Normalize(full);
+  }
+  // basename() ignores a drive prefix, but not the server/share in a UNC root.
+  const bool has_drive = kWindows && full.size() >= 2 &&
+                         IsWindowsDeviceRoot(full[0]) && full[1] == ':';
+  const size_t drive = has_drive ? 2 : 0;
+  size_t end = full.size();
+  while (end > drive && IsPathSeparator(full[end - 1])) end--;
+  size_t start = end;
+  while (start > drive && !IsPathSeparator(full[start - 1])) start--;
+
+  size_t root = RootLength(full);
+  if (kWindows && full.size() >= 2 && IsPathSeparator(full[0]) &&
+      IsPathSeparator(full[1])) {
+    // Unlike resolved walker paths, callback paths can have an incomplete UNC
+    // prefix. dirname() recognizes a UNC root only with both server and share.
+    size_t server_end = 2;
+    while (server_end < full.size() && !IsPathSeparator(full[server_end])) {
+      server_end++;
+    }
+    size_t share_start = server_end;
+    while (share_start < full.size() && IsPathSeparator(full[share_start])) {
+      share_start++;
+    }
+    root = 1;
+    if (server_end > 2 && share_start < full.size()) {
+      root = share_start;
+      while (root < full.size() && !IsPathSeparator(full[root])) root++;
+      if (root < full.size()) {
+        root++;
+      } else if (joined) {
+        // path.join() terminates a bare UNC share root with a separator.
+        full += kSep;
+        root++;
+      }
+    }
+  }
+  size_t parent_end = end;
+  while (parent_end > root && !IsPathSeparator(full[parent_end - 1])) {
+    parent_end--;
+  }
+  parent_end = parent_end > root ? parent_end - 1 : root;
+  // POSIX dirname('//name') keeps both leading separators.
+  if (!kWindows && root != 0 && parent_end == 1 && start == 2) {
+    parent_end = 2;
+  }
+  return {full.substr(start, end - start),
+          parent_end == 0 ? "." : full.substr(0, parent_end)};
+}
+
 // The pimpl the public Walk hands its calls to.
 class WalkerImpl : public WalkerState {
  public:

src/glob/glob_walker.h

@@ -52,6 +52,15 @@ struct WalkEntry {
   int type = UV_DIRENT_UNKNOWN;
 };
 
+struct DirentPath {
+  std::string name;
+  std::string parent;
+};
+
+// Split before encoding: path operations must preserve filesystem bytes.
+// An empty cwd leaves path as given, for exclude callbacks' lstat inputs.
+DirentPath GetDirentPath(const std::string& cwd, const std::string& path);
+
 // One call a walk off the main thread needs made to the exclude callback,
 // which only that thread can do. A directory's questions are recorded in
 // the order the callback would see them, together with what decides