Serialize ints below -2**31 through the long path; add serializer property tests

.pre-commit-config.yaml

@@ -30,6 +30,7 @@ repos:
     hooks:
     -   id: mypy
         additional_dependencies:
+          - hypothesis
           - pytest
           - types-pywin32
           - types-gevent

news/444.bugfix.rst

@@ -0,0 +1 @@
+Integers below ``-2**31`` are now serialized through the long-integer path instead of failing with ``struct.error``.

pyproject.toml

@@ -38,6 +38,7 @@ classifiers = [
 [project.optional-dependencies]
 testing = [
     "pre-commit",
+    "hypothesis",
     "pytest>8.0",
     "pytest-timeout",
     "tox",

src/execnet/gateway_base.py

@@ -1337,6 +1337,7 @@ def bchr(n: int) -> bytes:
 DUMPFORMAT_VERSION = bchr(2)
 
 FOUR_BYTE_INT_MAX = 2147483647
+FOUR_BYTE_INT_MIN = -2147483648
 
 FLOAT_FORMAT = "!d"
 FLOAT_FORMAT_SIZE = struct.calcsize(FLOAT_FORMAT)
@@ -1693,7 +1694,7 @@ def _write_byte_sequence(self, bytes_: bytes) -> None:
         self._write(bytes_)
 
     def _save_integral(self, i: int, short_op: bytes, long_op: bytes) -> None:
-        if i <= FOUR_BYTE_INT_MAX:
+        if FOUR_BYTE_INT_MIN <= i <= FOUR_BYTE_INT_MAX:
             self._write(short_op)
             self._write_int4(i)
         else:
@@ -1715,9 +1716,12 @@ def save_complex(self, cpx: complex) -> None:
         self._write(struct.pack(COMPLEX_FORMAT, cpx.real, cpx.imag))
 
     def _write_int4(
-        self, i: int, error: str = "int must be less than %i" % (FOUR_BYTE_INT_MAX,)
+        self,
+        i: int,
+        error: str = "int must be within [%i, %i]"
+        % (FOUR_BYTE_INT_MIN, FOUR_BYTE_INT_MAX),
     ) -> None:
-        if i > FOUR_BYTE_INT_MAX:
+        if not FOUR_BYTE_INT_MIN <= i <= FOUR_BYTE_INT_MAX:
             raise DumpError(error)
         self._write(struct.pack("!i", i))
 

testing/test_gateway.py

@@ -4,13 +4,15 @@
 
 from __future__ import annotations
 
+import importlib.util
 import os
 import pathlib
 import shutil
 import signal
 import sys
 from collections.abc import Callable
 from textwrap import dedent
+from types import ModuleType
 
 import pytest
 
@@ -32,6 +34,21 @@
 )
 
 
+def import_fresh(path: pathlib.Path) -> ModuleType:
+    """Execute the file at path as a new module object.
+
+    The module is not put into sys.modules and sys.path is untouched, so every
+    call gets a module whose __file__ is the given path. A plain ``import``
+    would hand later tests the module cached by an earlier one, whose file
+    may be gone.
+    """
+    spec = importlib.util.spec_from_file_location(path.stem, path)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
 class TestBasicGateway:
     def test_correct_setup(self, gw: Gateway) -> None:
         assert gw.hasreceiver()
@@ -112,7 +129,7 @@ def test_remote_exec_module(self, tmp_path: pathlib.Path, gw: Gateway) -> None:
         assert name == 2
 
     def test_remote_exec_module_is_removed(
-        self, gw: Gateway, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch
+        self, gw: Gateway, tmp_path: pathlib.Path
     ) -> None:
         remotetest = tmp_path / "remote.py"
         remotetest.write_text(
@@ -129,8 +146,8 @@ def remote():
             )
         )
 
-        monkeypatch.syspath_prepend(tmp_path)
-        import remote  # type: ignore[import-not-found]
+        remote = import_fresh(remotetest)
+        assert remote.__file__ == str(remotetest)
 
         ch = gw.remote_exec(remote)
         # simulate sending the code to a remote location that does not have
@@ -145,10 +162,7 @@ def remote():
         assert result is True
 
     def test_remote_exec_module_with_traceback(
-        self,
-        gw: Gateway,
-        tmp_path: pathlib.Path,
-        monkeypatch: pytest.MonkeyPatch,
+        self, gw: Gateway, tmp_path: pathlib.Path
     ) -> None:
         remotetestpy = tmp_path / "remotetest.py"
         remotetestpy.write_text(
@@ -163,14 +177,13 @@ def run_me(channel=None):
             )
         )
 
-        monkeypatch.syspath_prepend(tmp_path)
-        import remotetest  # type: ignore[import-not-found]
+        remotetest = import_fresh(remotetestpy)
 
         ch = gw.remote_exec(remotetest)
         try:
             ch.receive()
         except execnet.gateway_base.RemoteError as e:
-            assert 'remotetest.py", line 3, in run_me' in str(e)
+            assert f'{remotetestpy}", line 3, in run_me' in str(e)
             assert "ValueError: me" in str(e)
         finally:
             ch.close()
@@ -179,7 +192,7 @@ def run_me(channel=None):
         try:
             ch.receive()
         except execnet.gateway_base.RemoteError as e:
-            assert 'remotetest.py", line 3, in run_me' in str(e)
+            assert f'{remotetestpy}", line 3, in run_me' in str(e)
             assert "ValueError: me" in str(e)
         finally:
             ch.close()

testing/test_serializer.py

@@ -77,93 +77,48 @@ def load(py3: PythonWrapper):
     return py3.load
 
 
-simple_tests = [
-    # expected before/after repr
-    ("int", "4"),
-    ("float", "3.25"),
-    ("complex", "(1.78+3.25j)"),
-    ("list", "[1, 2, 3]"),
-    ("tuple", "(1, 2, 3)"),
-    ("dict", "{(1, 2, 3): 32}"),
+# expression evaluated by the dumping interpreter, loaded type name, loaded repr
+roundtrip_expectations = [
+    pytest.param("4", "int", "4", id="int"),
+    pytest.param("-4", "int", "-4", id="negative-int"),
+    pytest.param("2**31 - 1", "int", "2147483647", id="int32-max"),
+    pytest.param("-(2**31)", "int", "-2147483648", id="int32-min"),
+    pytest.param("2**31", "int", "2147483648", id="above-int32-max"),
+    pytest.param("-(2**31) - 1", "int", "-2147483649", id="below-int32-min"),
+    pytest.param(
+        "9223372036854775807324234", "int", "9223372036854775807324234", id="big-int"
+    ),
+    pytest.param(
+        "-9223372036854775807324234",
+        "int",
+        "-9223372036854775807324234",
+        id="big-negative-int",
+    ),
+    pytest.param("3.25", "float", "3.25", id="float"),
+    pytest.param("(1.78+3.25j)", "complex", "(1.78+3.25j)", id="complex"),
+    pytest.param("True", "bool", "True", id="bool"),
+    pytest.param("None", "NoneType", "None", id="none"),
+    pytest.param("b'hi'", "bytes", "b'hi'", id="bytes"),
+    pytest.param("'xyz'", "str", "'xyz'", id="str"),
+    pytest.param("u'hi'", "str", "'hi'", id="unicode-prefix-str"),
+    pytest.param("[1, 2, 3]", "list", "[1, 2, 3]", id="list"),
+    pytest.param("(1, 2, 3)", "tuple", "(1, 2, 3)", id="tuple"),
+    pytest.param("(1, (), 3)", "tuple", "(1, (), 3)", id="tuple-nested-empty"),
+    pytest.param("{(1, 2, 3): 32}", "dict", "{(1, 2, 3): 32}", id="dict-tuple-key"),
+    # one element only: multi-element set order is not part of the format
+    pytest.param("{1}", "set", "{1}", id="set-single"),
+    pytest.param("set()", "set", "set()", id="set-empty"),
+    pytest.param(
+        "frozenset((1, 2, 3))", "frozenset", "frozenset({1, 2, 3})", id="frozenset"
+    ),
 ]
 
 
-@pytest.mark.parametrize(["tp_name", "repr"], simple_tests)
-def test_simple(tp_name, repr, dump, load) -> None:
-    p = dump(repr)
-    tp, v = load(p)
+@pytest.mark.parametrize(["expr", "tp_name", "expected"], roundtrip_expectations)
+def test_roundtrip(expr: str, tp_name: str, expected: str, dump, load) -> None:
+    tp, v = load(dump(expr))
     assert tp == tp_name
-    assert v == repr
-
-
-def test_set(load, dump) -> None:
-    p = dump("set((1, 2, 3))")
-
-    tp, v = load(p)
-    assert tp == "set"
-    # assert v == "{1, 2, 3}" # ordering prevents this assertion
-    assert v.startswith("{") and v.endswith("}")
-    assert "1" in v and "2" in v and "3" in v
-    p = dump("set()")
-    tp, v = load(p)
-    assert tp == "set"
-    assert v == "set()"
-
-
-def test_frozenset(load, dump):
-    p = dump("frozenset((1, 2, 3))")
-    tp, v = load(p)
-    assert tp == "frozenset"
-    assert v == "frozenset({1, 2, 3})"
-
-
-def test_long(load, dump) -> None:
-    really_big = "9223372036854775807324234"
-    p = dump(really_big)
-    tp, v = load(p)
-    assert tp == "int"
-    assert v == really_big
-
-
-def test_bytes(dump, load) -> None:
-    p = dump("b'hi'")
-    tp, v = load(p)
-    assert tp == "bytes"
-    assert v == "b'hi'"
-
-
-def test_str(dump, load) -> None:
-    p = dump("'xyz'")
-    tp, s = load(p)
-    assert tp == "str"
-    assert s == "'xyz'"
-
-
-def test_unicode(load, dump) -> None:
-    p = dump("u'hi'")
-    tp, s = load(p)
-    assert tp == "str"
-    assert s == "'hi'"
-
-
-def test_bool(dump, load) -> None:
-    p = dump("True")
-    tp, s = load(p)
-    assert s == "True"
-    assert tp == "bool"
-
-
-def test_none(dump, load) -> None:
-    p = dump("None")
-    _tp, s = load(p)
-    assert s == "None"
-
-
-def test_tuple_nested_with_empty_in_between(dump, load) -> None:
-    p = dump("(1, (), 3)")
-    tp, s = load(p)
-    assert tp == "tuple"
-    assert s == "(1, (), 3)"
+    assert v == expected
 
 
 def test_py2_string_loads() -> None:

testing/test_serializer_hypothesis.py

@@ -0,0 +1,73 @@
+"""Property-based roundtrip tests for the execnet serializer."""
+
+from __future__ import annotations
+
+import struct
+from typing import Any
+
+from hypothesis import given
+from hypothesis import strategies as st
+
+import execnet
+
+hashable_scalars = (
+    st.none()
+    | st.booleans()
+    | st.integers()
+    | st.floats()
+    | st.complex_numbers()
+    | st.text(max_size=20)
+    | st.binary(max_size=20)
+)
+
+hashables = st.recursive(
+    hashable_scalars,
+    lambda children: (
+        st.tuples(children, children) | st.frozensets(children, max_size=4)
+    ),
+    max_leaves=8,
+)
+
+values = st.recursive(
+    hashables,
+    lambda children: (
+        st.lists(children, max_size=4)
+        | st.lists(children, max_size=4).map(tuple)
+        | st.dictionaries(hashables, children, max_size=4)
+        | st.sets(hashables, max_size=4)
+    ),
+    max_leaves=16,
+)
+
+
+def canonical(obj: Any) -> object:
+    """Map a value to a structure that compares equal only for identical input.
+
+    Plain ``==`` is not enough for a roundtrip check: ``1 == 1.0 == True``,
+    ``0.0 == -0.0`` and ``nan != nan``. Floats are compared by bit pattern
+    and every node carries its exact type.
+    """
+    tp = type(obj).__name__
+    if isinstance(obj, float):
+        return (tp, struct.pack("!d", obj))
+    if isinstance(obj, complex):
+        return (tp, struct.pack("!dd", obj.real, obj.imag))
+    if isinstance(obj, (list, tuple)):
+        return (tp, [canonical(x) for x in obj])
+    if isinstance(obj, dict):
+        return (tp, [(canonical(k), canonical(v)) for k, v in obj.items()])
+    if isinstance(obj, (set, frozenset)):
+        return (tp, sorted(repr(canonical(x)) for x in obj))
+    return (tp, obj)
+
+
+@given(values)
+def test_roundtrip(value: object) -> None:
+    assert canonical(execnet.loads(execnet.dumps(value))) == canonical(value)
+
+
+@given(st.integers())
+def test_int_roundtrip(value: int) -> None:
+    loaded = execnet.loads(execnet.dumps(value))
+    assert type(loaded) is int
+    assert loaded == value

tox.ini

@@ -7,6 +7,7 @@ usedevelop=true
 setenv =
     PYTHONWARNDEFAULTENCODING = 1
 deps=
+    hypothesis
     pytest
     pytest-timeout
 passenv = GITHUB_ACTIONS, HOME, USER, XDG_*