From e9cdcde40c9f0efc69bc7928aca99db49814c1a7 Mon Sep 17 00:00:00 2001 From: Sun Haoyuan <58774225+Shy7777@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:36:41 +0800 Subject: [PATCH] Python: preserve Unicode file store display paths --- .../agent_framework/_harness/_file_access.py | 22 +++++++------ .../tests/core/test_harness_file_access.py | 32 +++++++++++++++++++ 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/python/packages/core/agent_framework/_harness/_file_access.py b/python/packages/core/agent_framework/_harness/_file_access.py index c7c48dae495..6191bb98f7d 100644 --- a/python/packages/core/agent_framework/_harness/_file_access.py +++ b/python/packages/core/agent_framework/_harness/_file_access.py @@ -1113,6 +1113,7 @@ async def list_children(self, directory: str = "") -> list[FileStoreEntry]: prefix = _normalize_relative_path(directory, is_directory=True).lower() if prefix and not prefix.endswith("/"): prefix += "/" + prefix_depth = prefix.count("/") async with self._lock: entries = [(key, display) for key, (display, _) in self._files.items()] files: list[str] = [] @@ -1121,18 +1122,18 @@ async def list_children(self, directory: str = "") -> list[FileStoreEntry]: for key, display in entries: if not key.startswith(prefix): continue - remainder = key[len(prefix) :] - separator_index = remainder.find("/") - if separator_index == -1: - # ``display`` is the original-case normalized path; strip the - # directory prefix using the same length we matched on ``key``. - files.append(display[len(prefix) :]) - elif separator_index > 0: - segment_key = remainder[:separator_index] + # Unicode lowercasing can change character counts, so key offsets + # cannot be used to slice the original display path. + remainder = display.split("/", prefix_depth)[-1] + segment, separator, _ = remainder.partition("/") + if not separator: + files.append(remainder) + elif segment: + segment_key = segment.lower() if segment_key in seen_dirs: continue seen_dirs.add(segment_key) - directories.append(display[len(prefix) : len(prefix) + separator_index]) + directories.append(segment) results: list[FileStoreEntry] = [FileStoreEntry(name, FileStoreEntry.DIRECTORY) for name in directories] results.extend(FileStoreEntry(name, FileStoreEntry.FILE) for name in files) return results @@ -1165,6 +1166,7 @@ async def search( prefix = _normalize_relative_path(directory, is_directory=True).lower() if prefix and not prefix.endswith("/"): prefix += "/" + prefix_depth = prefix.count("/") search_pattern = _compile_search_regex(regex_pattern) async with self._lock: @@ -1178,7 +1180,7 @@ def scan() -> list[FileSearchResult]: relative_key = key[len(prefix) :] if not recursive and "/" in relative_key: continue - relative_display = display[len(prefix) :] + relative_display = display.split("/", prefix_depth)[-1] if not _matches_glob(relative_display, glob_pattern): continue result = AgentFileStore.scan_content(relative_display, file_content, search_pattern) diff --git a/python/packages/core/tests/core/test_harness_file_access.py b/python/packages/core/tests/core/test_harness_file_access.py index 9cbffc96abf..8c01b1c6146 100644 --- a/python/packages/core/tests/core/test_harness_file_access.py +++ b/python/packages/core/tests/core/test_harness_file_access.py @@ -175,6 +175,38 @@ async def test_in_memory_store_round_trips_files() -> None: assert sorted(await _list_files(store)) == [] +@pytest.mark.parametrize("directory", ["İstanbul", "i\u0307stanbul", "İSTANBUL/"]) +async def test_in_memory_store_lists_unicode_paths_without_truncation(directory: str) -> None: + """Length-changing lowercasing must not alter the displayed path components.""" + store = InMemoryAgentFileStore() + await store.write("İstanbul/Notes.md", "memory") + await store.write("İstanbul/İzmir/plan.md", "nested memory") + await store.write("i\u0307stanbul/i\u0307zmir/other.md", "another memory") + + assert await _list_dirs(store) == ["İstanbul"] + assert await _list_files(store, directory) == ["Notes.md"] + assert await _list_dirs(store, directory) == ["İzmir"] + assert sorted(await _list_files(store, f"{directory.rstrip('/')}/İzmir")) == ["other.md", "plan.md"] + assert await store.read(f"{directory.rstrip('/')}/notes.MD") == "memory" + + +@pytest.mark.parametrize("directory", ["İstanbul", "i\u0307stanbul", "İSTANBUL/"]) +async def test_in_memory_store_search_preserves_unicode_relative_paths(directory: str) -> None: + """Search results must retain readable paths after matching a Unicode directory.""" + store = InMemoryAgentFileStore() + await store.write("İstanbul/Notes.md", "memory") + await store.write("İstanbul/İzmir/plan.md", "memory") + await store.write("other/notes.md", "memory") + + direct = await store.search(directory, "memory", "Notes.md") + assert [result.file_name for result in direct] == ["Notes.md"] + + recursive = await store.search(directory, "memory", recursive=True) + assert {result.file_name for result in recursive} == {"Notes.md", "İzmir/plan.md"} + for result in recursive: + assert await store.read(f"{directory.rstrip('/')}/{result.file_name}") == "memory" + + async def test_in_memory_store_search_returns_matches_with_snippets() -> None: """The in-memory store should search file content case-insensitively and respect glob filters.""" store = InMemoryAgentFileStore()