A dictionary assigning new columns to groups: map a new column's qualified
title (its key in data, e.g. "Assays▸Cu") to the qualified title of the group it belongs to.
-To move or ungroup an existing column, use :meth:update_column_metadata instead — a group change is
-metadata-only and does not require re-uploading data.
column_metadata_updates (in the same update) or
+:meth:update_column_metadata (on its own).
new_groups
+ list[GroupDefinition] | None
+ Definitions of new groups to create in the same update, so a group can be created and +populated (by new columns or by moving existing ones) atomically in a single new version.
+None
+ column_metadata_updates
+ dict[str, ColumnMetadataUpdate] | None
+ Metadata updates (unit, tags and/or group) for existing columns, applied in
+the same update and keyed by each column's current title. Set ColumnMetadataUpdate(group=...) to
+move a column into a group (or "" to ungroup it). When a moved column's data is re-uploaded (it is in
+update_columns, named by its current title), key that data by its destination title, e.g.
+Assays▸Cu after moving Cu into Assays. A column cannot also be in delete_columns.
None
+ A dictionary assigning new columns to groups: map a new column's qualified
title (its key in data, e.g. "Assays▸Cu") to the qualified title of the group it belongs to.
-To move or ungroup an existing column, use :meth:update_column_metadata instead — a group change is
-metadata-only and does not require re-uploading data.
column_metadata_updates (in the same update) or
+:meth:update_column_metadata (on its own).
new_groups
+ list[GroupDefinition] | None
+ Definitions of new groups to create in the same update, so a group can be created and +populated (by new columns or by moving existing ones) atomically in a single new version.
+None
+ column_metadata_updates
+ dict[str, ColumnMetadataUpdate] | None
+ Metadata updates (unit, tags and/or group) for existing columns, applied in
+the same update and keyed by each column's current title. Set ColumnMetadataUpdate(group=...) to
+move a column into a group (or "" to ungroup it). When a moved column's data is re-uploaded (it is in
+update_columns, named by its current title), key that data by its destination title, e.g.
+Assays▸Cu after moving Cu into Assays. A column cannot also be in delete_columns.
None
+ Create, update, and/or delete column groups on a block model.
-This method manages group definitions without requiring data upload or cache configuration. Any
-combination of new, update and delete can be supplied in a single call.
Create, update, and/or delete column groups, and move existing columns between groups.
+This method manages group definitions and group membership without requiring data upload or cache
+configuration. Any combination of new, update, delete and move_columns can be supplied
+in a single call, producing a single new version.
Groups are addressed by their qualified title (a bare title for a top-level group, or segments
joined by ▸ for a nested group). To assign a new column to a group, use the column_groups
-parameter on the column methods; to move or ungroup an existing column, use
-:meth:update_column_metadata. To resolve a written group back to its server-assigned UUID and
+parameter on the column methods; to move or ungroup an existing column, use move_columns. To resolve a written group back to its server-assigned UUID and
resolved policy, use the helpers on the returned :class:~evo.blockmodels.data.Version, e.g.
:meth:~evo.blockmodels.data.Version.group_by_qualified_title.
move_columns
+ dict[str, str] | None
+ A dictionary mapping an existing column's current title (qualified
+group▸title if it is currently grouped, plain otherwise) to the qualified title of the group to
+move it into, or "" to ungroup it. Only the column's group changes; the target group may be one
+created in new in the same call. To change a column's unit or tags, use
+:meth:update_column_metadata.
None
+ comment
diff --git a/packages/evo-files/src/evo/files/io.py b/packages/evo-files/src/evo/files/io.py
index ec71ee795..4d79ab674 100644
--- a/packages/evo-files/src/evo/files/io.py
+++ b/packages/evo-files/src/evo/files/io.py
@@ -16,6 +16,7 @@
from uuid import UUID
from evo.common import APIConnector, Environment, ICache
+from evo.common._types import resolve_path
from evo.common.io import Download, Upload
from .data import FileMetadata
@@ -176,7 +177,7 @@ def _get_cache_location(self, cache: ICache) -> Path:
/ str(self.metadata.id)
/ self.metadata.version_id
)
- download_dir.mkdir(parents=True, exist_ok=True)
+ resolve_path(download_dir).mkdir(parents=True, exist_ok=True)
return download_dir / _make_name_safe(self.metadata.name)
async def get_download_url(self) -> str:
diff --git a/packages/evo-files/tests/test_file_io.py b/packages/evo-files/tests/test_file_io.py
index 70d1fe314..fbf54b96a 100644
--- a/packages/evo-files/tests/test_file_io.py
+++ b/packages/evo-files/tests/test_file_io.py
@@ -10,10 +10,14 @@
# limitations under the License.
import json
+import os
+import unittest
+from unittest import mock
from uuid import UUID
from data import load_test_data
from evo.common import ServiceUser
+from evo.common._types import resolve_path
from evo.common.test_tools import TestWithConnector, TestWithDownloadHandler, TestWithUploadHandler, utc_datetime
from evo.files import FileAPIDownload, FileAPIUpload, FileMetadata
@@ -109,6 +113,31 @@ async def test_download_to_cache(self) -> None:
self.assertEqual(TEST_DATA, dest.read_bytes())
self.assert_download_requests(INITIAL_URL)
+ @unittest.skipUnless(os.name == "nt", "Windows long-path regression")
+ async def test_download_to_cache_long_path(self) -> None:
+ cache_location = self.cache.root / ("a" * 70) / ("b" * 70) / ("c" * 70)
+ resolved_cache_location = resolve_path(cache_location)
+ resolved_cache_location.mkdir(parents=True, exist_ok=True)
+ cache = mock.Mock()
+ cache.get_location.return_value = cache_location
+
+ expected = resolve_path(cache_location / str(FILE_ID) / VERSION_ID / self.metadata.name)
+ self.assertGreater(len(str(expected)), 259)
+
+ original_mkdir = type(resolved_cache_location).mkdir
+
+ def mkdir_with_legacy_limit(path, *args, **kwargs):
+ if len(str(path)) > 259 and not str(path).startswith("\\\\?\\"):
+ raise OSError("Path exceeds the legacy Windows limit")
+ return original_mkdir(path, *args, **kwargs)
+
+ with mock.patch.object(type(resolved_cache_location), "mkdir", mkdir_with_legacy_limit):
+ dest = await self.download.download_to_cache(cache, self.transport)
+
+ self.assertEqual(expected, resolve_path(dest))
+ self.assertEqual(TEST_DATA, dest.read_bytes())
+ self.assert_download_requests(INITIAL_URL)
+
class TestFileAPIUpload(TestWithConnector, TestWithUploadHandler):
def setUp(self) -> None:
diff --git a/packages/evo-sdk-common/src/evo/common/_types.py b/packages/evo-sdk-common/src/evo/common/_types.py
index 578d86830..fe7ad6000 100644
--- a/packages/evo-sdk-common/src/evo/common/_types.py
+++ b/packages/evo-sdk-common/src/evo/common/_types.py
@@ -27,7 +27,7 @@
PathLike: TypeAlias = str | os.PathLike | Path
-MAX_WINDOWS_PATH_LEN = 259
+MAX_WINDOWS_PATH_LEN = 32766
FILE_NAME_TOO_LONG_ERR_NO = 63
@@ -63,6 +63,8 @@ def _check_path_length(path: Path) -> None:
def resolve_path(path: PathLike, *, check_path_length: bool = False) -> Path:
"""Resolve the given path and check for maximum path length.
+ Windows paths use the extended-length form so filesystem operations do not depend on the system long-path setting.
+
:param path: The path to resolve.
:param check_path_length: If True, check the path length.
@@ -77,6 +79,15 @@ def resolve_path(path: PathLike, *, check_path_length: bool = False) -> Path:
except TypeError:
raise TypeError(f"Expected a PathLike object, got {path!r}")
+ if os.name == "nt":
+ absolute_path = os.path.abspath(path)
+ if not absolute_path.startswith(("\\\\?\\", "\\\\.\\")):
+ if absolute_path.startswith("\\\\"):
+ absolute_path = "\\\\?\\UNC\\" + absolute_path[2:]
+ else:
+ absolute_path = "\\\\?\\" + absolute_path
+ path = Path(absolute_path)
+
resolved = path.resolve()
if check_path_length:
_check_path_length(resolved)
diff --git a/packages/evo-sdk-common/tests/common/io/test_http.py b/packages/evo-sdk-common/tests/common/io/test_http.py
index 0c23aa8ae..925c6431f 100644
--- a/packages/evo-sdk-common/tests/common/io/test_http.py
+++ b/packages/evo-sdk-common/tests/common/io/test_http.py
@@ -12,6 +12,7 @@
from pathlib import Path
from unittest import mock
+from evo.common._types import resolve_path
from evo.common.connector import APIConnector
from evo.common.io import HTTPSource
from evo.common.test_tools import TestWithDownloadHandler
@@ -78,6 +79,17 @@ async def test_download_file(self) -> None:
self.assertTrue(test_data_file.exists())
self.assertEqual(TEST_DATA, test_data_file.read_bytes())
+ async def test_download_file_long_path(self) -> None:
+ test_data_file = self.CACHE_DIR / ("a" * 100) / ("b" * 100) / "download.csv"
+ resolved = resolve_path(test_data_file)
+ resolved.parent.mkdir(parents=True)
+ self.assertGreater(len(str(test_data_file)), 259)
+
+ await HTTPSource.download_file(str(test_data_file), self.url_generator.get_new_url, self.transport)
+
+ self.assertEqual(TEST_DATA, resolved.read_bytes())
+ self.assertEqual([resolved], list(resolved.parent.iterdir()))
+
async def test_download_file_uses_concurrently_published_file(self) -> None:
"""Test that a concurrent process publishing the destination is accepted."""
test_data_file = self.CACHE_DIR / "test_data_download_concurrent.csv"
diff --git a/packages/evo-sdk-common/tests/common/io/test_storage.py b/packages/evo-sdk-common/tests/common/io/test_storage.py
index 79692807e..33045e3bd 100644
--- a/packages/evo-sdk-common/tests/common/io/test_storage.py
+++ b/packages/evo-sdk-common/tests/common/io/test_storage.py
@@ -23,6 +23,7 @@
from parameterized import parameterized_class
from evo.common import RequestMethod
+from evo.common._types import resolve_path
from evo.common.connector import APIConnector
from evo.common.io.storage import BlockList, StorageBlock, StorageDestination
from evo.common.test_tools import TestWithUploadHandler, utc_datetime
@@ -198,6 +199,17 @@ async def test_upload_file(self) -> None:
uploaded_data = await self.handler.get_committed()
self.assertEqual(test_data_file.read_bytes(), uploaded_data)
+ async def test_upload_file_long_path(self) -> None:
+ test_data_file = self.CACHE_DIR / ("a" * 100) / ("b" * 100) / "upload.csv"
+ resolved = resolve_path(test_data_file)
+ resolved.parent.mkdir(parents=True)
+ resolved.write_bytes(TEST_DATA)
+ self.assertGreater(len(str(test_data_file)), 259)
+
+ await StorageDestination.upload_file(str(test_data_file), self.url_generator.get_new_url, self.transport)
+
+ self.assertEqual(TEST_DATA, await self.handler.get_committed())
+
@mock.patch("evo.common.io.http.APIConnector", wraps=APIConnector)
async def test_additional_headers_passed_to_connector(self, mock_api_connector: mock.Mock) -> None:
additional_headers = {"X-Custom-Header": "custom-value", "X-Another-Header": "another-value"}
diff --git a/packages/evo-sdk-common/tests/common/test_types.py b/packages/evo-sdk-common/tests/common/test_types.py
new file mode 100644
index 000000000..2bb35ef17
--- /dev/null
+++ b/packages/evo-sdk-common/tests/common/test_types.py
@@ -0,0 +1,42 @@
+import os
+import unittest
+from pathlib import Path
+from tempfile import NamedTemporaryFile, TemporaryDirectory
+
+from evo.common._types import resolve_path
+
+
+@unittest.skipUnless(os.name == "nt", "Windows extended-length paths")
+class TestWindowsPaths(unittest.TestCase):
+ def test_long_path_can_be_opened(self) -> None:
+ with TemporaryDirectory() as directory:
+ path = Path(directory) / ("a" * 100) / ("b" * 100) / ("c" * 100) / "data.bin"
+ resolved = resolve_path(path, check_path_length=True)
+ self.assertTrue(str(resolved).startswith("\\\\?\\"))
+ resolved.parent.mkdir(parents=True)
+ resolved.write_bytes(b"transfer data")
+ self.assertEqual(b"transfer data", resolved.read_bytes())
+ self.assertEqual(resolved, resolve_path(resolved, check_path_length=True))
+
+ def test_short_path_supports_long_sibling_temporary_file(self) -> None:
+ with TemporaryDirectory() as directory:
+ path = Path(directory) / ("a" * 100)
+ path /= "b" * (250 - len(str(path)))
+ path /= "x"
+ self.assertLess(len(str(path)), 260)
+ resolved = resolve_path(path, check_path_length=True)
+ self.assertTrue(str(resolved).startswith("\\\\?\\"))
+ resolved.parent.mkdir(parents=True)
+ with NamedTemporaryFile(dir=resolved.parent) as temporary_file:
+ self.assertGreater(len(temporary_file.name), 259)
+ temporary_file.write(b"transfer data")
+
+ def test_unc_path(self) -> None:
+ path = resolve_path(r"\\server\share\folder\data.bin")
+ self.assertEqual(r"\\?\UNC\server\share\folder\data.bin", str(path))
+ self.assertEqual(path, resolve_path(path))
+
+
+class TestResolvePath(unittest.TestCase):
+ def test_relative_path(self) -> None:
+ self.assertEqual(resolve_path("data.bin"), resolve_path(Path("folder") / ".." / "data.bin"))