From a38cde966dbf9420bcebd3111acfa71081c32558 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Thu, 19 Nov 2026 00:52:47 +0000 Subject: [PATCH] [SPARK-62056][CONNECT][PYTHON] Take the arguments of the methods of the maps and repeated containers of the Rust protobuf runtime as upb's, with upb's errors The methods of the containers of the drop-in's Rust protobuf runtime (pyspark.sql.connect.proto) take their arguments as upb's do, with upb's errors, where they had PyO3's keyword arguments and errors: - Maps: MergeFrom, clear, GetEntryClass and get_or_create are upb's C methods of one argument or of none, positional only, with CPython's errors naming upb's class, ScalarMapContainer or MessageMapContainer ("ScalarMapContainer.MergeFrom() takes exactly one argument (0 given)"). setdefault takes no keyword arguments, refuses a map of messages before it counts its arguments, and counts them as PyArg_UnpackTuple does. get parses its key and default as PyArg_ParseTupleAndKeywords("O|O") does. pop, popitem, keys, values and items are collections.abc's MutableMapping's and Mapping's, as upb's are, so their errors are Python's. - Repeated containers: append, extend, remove and MergeFrom take one positional argument, insert and pop parse theirs as PyArg_ParseTuple's "nO" and "|n" (an index of its __index__, with "Python int too large to convert to C ssize_t"), and __deepcopy__ takes no keyword argument, as upb's. Before, append(value=...), insert(key=..., value=...), pop(key=...), remove(elem=...) and MergeFrom(other=...) were taken. - A keyword argument of upb's METH_VARARGS methods (setdefault, insert, pop, __deepcopy__) is CPython's error of the method of its class when it is called of its object, and of the method only when CPython calls it as a bound method, as it calls each before Python 3.12 while a trace or a profile function is set (coverage's): the runtime's error names the method only then too. - docs/compatibility.md: the arguments of the containers' methods, and why a dunder method called directly with the wrong arguments keeps CPython's error of a slot wrapper, and a keyword argument of a bound method object of a METH_VARARGS method names the class; a reflowed paragraph. Co-authored-by: Isaac --- crates/pyspark-rs/src/pbmsg/containers.rs | 568 ++++++++++++++++++---- docs/compatibility.md | 53 +- python/tests/test_proto_rust_offline.py | 156 ++++++ 3 files changed, 650 insertions(+), 127 deletions(-) diff --git a/crates/pyspark-rs/src/pbmsg/containers.rs b/crates/pyspark-rs/src/pbmsg/containers.rs index 78bf43df..41c54403 100644 --- a/crates/pyspark-rs/src/pbmsg/containers.rs +++ b/crates/pyspark-rs/src/pbmsg/containers.rs @@ -269,13 +269,216 @@ fn not_in_container() -> PyErr { PyValueError::new_err("remove(x): x not in container") } -/// A container of `value`'s items, by `collections.abc` `name` (such as `KeysView`) of `value`. -fn view<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult> { - value - .py() - .import("collections.abc")? - .getattr(name)? - .call1((value,)) +/// upb's errors of the arguments of a method `method` of a container of upb's class `class` +/// (`ScalarMapContainer`, `MessageMapContainer`, `RepeatedScalarContainer`, ...) that takes none +/// (`METH_NOARGS`), as CPython checks them: a keyword argument first. +fn no_arguments( + class: &str, + method: &str, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult<()> { + no_keywords(class, method, kwargs)?; + if !args.is_empty() { + return Err(PyTypeError::new_err(format!( + "{class}.{method}() takes no arguments ({} given)", + args.len() + ))); + } + Ok(()) +} + +/// upb's errors of the arguments of a method of a container that takes exactly one (`METH_O`), as +/// [`no_arguments`] gives them: the argument. +fn one_argument<'py>( + class: &str, + method: &str, + args: &Bound<'py, PyTuple>, + kwargs: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + no_keywords(class, method, kwargs)?; + if args.len() != 1 { + return Err(PyTypeError::new_err(format!( + "{class}.{method}() takes exactly one argument ({} given)", + args.len() + ))); + } + args.get_item(0) +} + +/// upb's error of a keyword argument of a method of a container that takes one or none +/// (`METH_O`, `METH_NOARGS`), which names the method of its class however it is called. +fn no_keywords(class: &str, method: &str, kwargs: Option<&Bound<'_, PyDict>>) -> PyResult<()> { + if kwargs.is_some_and(|k| !k.is_empty()) { + return Err(PyTypeError::new_err(format!( + "{class}.{method}() takes no keyword arguments" + ))); + } + Ok(()) +} + +/// upb's error of a keyword argument of a method of a container that takes positional arguments +/// (`METH_VARARGS`: `setdefault`, `insert`, `pop`, `__deepcopy__`), which CPython words by the +/// way that the method is called: called of its object (`map.setdefault(key="k")`), it names the +/// method of its class (`ScalarMapContainer.setdefault() takes no keyword arguments`), and called +/// as a bound method, it names the method only (`setdefault() takes no keyword arguments`). The +/// runtime's methods cannot tell the two apart, so the error names the method of its class, but +/// where CPython binds the method of every call: before 3.12, while a trace or a profile function +/// is set on the thread (as `coverage` and debuggers set one), CPython's `trace_call_function` +/// calls a method of a native type as a bound method, so that a profiler sees it. +fn no_keywords_of_positional( + py: Python<'_>, + class: &str, + method: &str, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult<()> { + if kwargs.is_none_or(|k| k.is_empty()) { + return Ok(()); + } + let bound = py.version_info() < (3, 12) && { + let sys = py.import("sys")?; + !sys.call_method0("gettrace")?.is_none() || !sys.call_method0("getprofile")?.is_none() + }; + Err(PyTypeError::new_err(if bound { + format!("{method}() takes no keyword arguments") + } else { + format!("{class}.{method}() takes no keyword arguments") + })) +} + +/// The `key` and `default` of upb's `get` of a map, which parses them as CPython's +/// `PyArg_ParseTupleAndKeywords(args, kwargs, "O|O", {"key", "default"})` does, with its errors +/// (of a "function", as the format names none): too many arguments, a missing `key`, an argument +/// given by name and position, then a keyword of no argument. +fn get_arguments<'py>( + args: &Bound<'py, PyTuple>, + kwargs: Option<&Bound<'py, PyDict>>, +) -> PyResult<(Bound<'py, PyAny>, Option>)> { + const NAMES: [&str; 2] = ["key", "default"]; + let nargs = args.len(); + let nkwargs = kwargs.map_or(0, |k| k.len()); + if nargs + nkwargs > NAMES.len() { + return Err(PyTypeError::new_err(format!( + "function takes at most {} {}arguments ({} given)", + NAMES.len(), + if nargs == 0 { "keyword " } else { "" }, + nargs + nkwargs + ))); + } + let keyword = |name: &str| -> PyResult>> { + match kwargs { + Some(kwargs) => kwargs.get_item(name), + None => Ok(None), + } + }; + let mut values: [Option>; 2] = [None, None]; + for (i, name) in NAMES.iter().enumerate() { + values[i] = if i < nargs { + Some(args.get_item(i)?) + } else { + keyword(name)? + }; + if values[i].is_none() && i == 0 { + return Err(PyTypeError::new_err( + "function missing required argument 'key' (pos 1)", + )); + } + } + for (i, name) in NAMES.iter().enumerate().take(nargs) { + if keyword(name)?.is_some() { + return Err(PyTypeError::new_err(format!( + "argument for function given by name ('{name}') and position ({})", + i + 1 + ))); + } + } + for key in kwargs.iter().flat_map(|k| k.keys()) { + if !key + .extract::() + .is_ok_and(|k| NAMES.contains(&k.as_str())) + { + return Err(PyTypeError::new_err(format!( + "'{}' is an invalid keyword argument for this function", + key.str()? + ))); + } + } + let [key, default] = values; + Ok((key.expect("the key, which is required"), default)) +} + +/// `arg` as an index of upb's `PyArg_ParseTuple` "n" format: of its `__index__`, with CPython's +/// errors ("'str' object cannot be interpreted as an integer", "Python int too large to convert to +/// C ssize_t"). +fn ssize_t_of(arg: &Bound<'_, PyAny>) -> PyResult { + let py = arg.py(); + // SAFETY: the pointers are of live objects; `PyNumber_Index` gives a new reference, owned by + // the `Bound` that wraps it, or null with an exception set. + unsafe { + let index = ffi::PyNumber_Index(arg.as_ptr()); + if index.is_null() { + return Err(PyErr::fetch(py)); + } + let index = Bound::from_owned_ptr(py, index); + let value = ffi::PyLong_AsSsize_t(index.as_ptr()); + match PyErr::take(py) { + Some(error) if value == -1 => Err(error), + _ => Ok(value), + } + } +} + +/// The index and the value of upb's `insert` of a repeated field of `class`, as it takes them of +/// `PyArg_ParseTuple(args, "nO")`: of no keyword argument (`METH_VARARGS`), exactly two, the index +/// of [`ssize_t_of`]. +fn insert_arguments<'py>( + class: &str, + args: &Bound<'py, PyTuple>, + kwargs: Option<&Bound<'py, PyDict>>, +) -> PyResult<(isize, Bound<'py, PyAny>)> { + no_keywords_of_positional(args.py(), class, "insert", kwargs)?; + if args.len() != 2 { + return Err(PyTypeError::new_err(format!( + "function takes exactly 2 arguments ({} given)", + args.len() + ))); + } + Ok((ssize_t_of(&args.get_item(0)?)?, args.get_item(1)?)) +} + +/// The index of upb's `pop` of a repeated field of `class`, as it takes it of +/// `PyArg_ParseTuple(args, "|n")`: of no keyword argument, at most one, -1 of none. +fn pop_argument( + class: &str, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult { + no_keywords_of_positional(args.py(), class, "pop", kwargs)?; + match args.len() { + 0 => Ok(-1), + 1 => ssize_t_of(&args.get_item(0)?), + given => Err(PyTypeError::new_err(format!( + "function takes at most 1 argument ({given} given)" + ))), + } +} + +/// The method `name` of `collections.abc`'s class `abc` (`Mapping`, `MutableMapping`) called of +/// `map` and of `args` and `kwargs`, as upb's maps have it: so its result, and the errors of its +/// arguments, are Python's (`MutableMapping.pop() missing 1 required positional argument: 'key'`). +fn abc_method( + map: &Bound<'_, MapContainer>, + abc: &str, + name: &str, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py = map.py(); + let method = py.import("collections.abc")?.getattr(abc)?.getattr(name)?; + let mut all = Vec::with_capacity(args.len() + 1); + all.push(map.as_any().clone()); + all.extend(args.iter()); + Ok(method.call(PyTuple::new(py, all)?, kwargs)?.unbind()) } /// The key and value of `pair`, as `for key, value in pairs` unpacks it. @@ -590,20 +793,15 @@ impl RepeatedScalar { } } -#[pymethods] impl RepeatedScalar { - fn append(slf: &Bound<'_, Self>, value: &Bound<'_, PyAny>) -> PyResult<()> { - Self::reify(slf)?; - let v = Self::field_of(slf).convert(value)?; - slf.borrow_mut().items.push(v); - Ok(()) - } + /// The name of upb's class of the container, which its errors of arguments name. + const UPB_CLASS: &'static str = "RepeatedScalarContainer"; /// As upb's: the values of `elem_seq` (of `PySequence_Fast`, so those of an iterator to its /// end first), each converted and appended in turn. A failure, of the iteration too, leaves /// the container of the length that it had: upb resizes it, so a position that Python code /// removed meanwhile holds the default. - fn extend(slf: &Bound<'_, Self>, elem_seq: &Bound<'_, PyAny>) -> PyResult<()> { + fn extend_values(slf: &Bound<'_, Self>, elem_seq: &Bound<'_, PyAny>) -> PyResult<()> { Self::reify(slf)?; let field = Self::field_of(slf); let start = slf.borrow().items.len(); @@ -615,11 +813,46 @@ impl RepeatedScalar { } extended } +} + +/// The methods take their arguments as upb's do, positional only, with upb's errors. +#[pymethods] +impl RepeatedScalar { + #[pyo3(signature = (*args, **kwargs))] + fn append( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + let value = &one_argument(Self::UPB_CLASS, "append", args, kwargs)?; + Self::reify(slf)?; + let v = Self::field_of(slf).convert(value)?; + slf.borrow_mut().items.push(v); + Ok(()) + } + + /// See [`RepeatedScalar::extend_values`]. + #[pyo3(signature = (*args, **kwargs))] + fn extend( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + let elem_seq = one_argument(Self::UPB_CLASS, "extend", args, kwargs)?; + Self::extend_values(slf, &elem_seq) + } /// As upb's: the position of `key` in the container as it is, then the value converted. A /// conversion that shortened the container leaves the position past its end (where upb's /// insert writes out of its bounds), so the value goes at its end. - fn insert(slf: &Bound<'_, Self>, key: isize, value: &Bound<'_, PyAny>) -> PyResult<()> { + #[pyo3(signature = (*args, **kwargs))] + fn insert( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + let (key, value) = insert_arguments(Self::UPB_CLASS, args, kwargs)?; + let value = &value; Self::reify(slf)?; let at = insert_position(key, slf.borrow().items.len()); let v = Self::field_of(slf).convert(value)?; @@ -629,8 +862,13 @@ impl RepeatedScalar { Ok(()) } - #[pyo3(signature = (key=-1))] - fn pop(slf: &Bound<'_, Self>, key: isize) -> PyResult> { + #[pyo3(signature = (*args, **kwargs))] + fn pop( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + let key = pop_argument(Self::UPB_CLASS, args, kwargs)?; let py = slf.py(); Self::reify(slf)?; let len = slf.borrow().items.len(); @@ -642,7 +880,13 @@ impl RepeatedScalar { /// As upb's: the first item that equals `elem`, as Python compares them, among as many items /// as the container had, each of the container as it is when it is compared (an `__eq__` may /// change it). - fn remove(slf: &Bound<'_, Self>, elem: &Bound<'_, PyAny>) -> PyResult<()> { + #[pyo3(signature = (*args, **kwargs))] + fn remove( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + let elem = &one_argument(Self::UPB_CLASS, "remove", args, kwargs)?; let py = slf.py(); Self::reify(slf)?; let len = slf.borrow().items.len(); @@ -692,16 +936,27 @@ impl RepeatedScalar { Ok(()) } - #[pyo3(name = "MergeFrom")] - fn merge_from(slf: &Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult<()> { - Self::extend(slf, other) + #[pyo3(name = "MergeFrom", signature = (*args, **kwargs))] + fn merge_from( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + let other = one_argument(Self::UPB_CLASS, "MergeFrom", args, kwargs)?; + Self::extend_values(slf, &other) } - /// A copy of the container, held by no message, as upb's `copy.deepcopy` gives one. - #[pyo3(signature = (memo=None))] - fn __deepcopy__(&self, memo: Option<&Bound<'_, PyAny>>) -> Self { - let _ = memo; - self.copy() + /// A copy of the container, held by no message, as upb's `copy.deepcopy` gives one: of any + /// positional arguments (the memo) and of no keyword argument, as upb's `METH_VARARGS`. + #[pyo3(signature = (*args, **kwargs))] + fn __deepcopy__( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult { + let _ = args; + no_keywords_of_positional(slf.py(), Self::UPB_CLASS, "__deepcopy__", kwargs)?; + Ok(slf.borrow().copy()) } fn __reduce__(&self, py: Python<'_>) -> PyResult> { @@ -884,8 +1139,26 @@ impl RepeatedComposite { PyMessage::merge_from_with(&child, value, budget)?; Ok(child) } + + /// The name of upb's class of the container, which its errors of arguments name. + const UPB_CLASS: &'static str = "RepeatedCompositeContainer"; + + /// As upb's: each value of the iterator of `elem_seq` appended as it comes, so a failure + /// leaves the values before it appended. One interrupt budget counts a step for each message + /// that the appends merge or copy, so that the extend of many messages lets another thread + /// take the GIL, and a signal interrupt it, as a copy does. + fn extend_values(slf: &Bound<'_, Self>, elem_seq: &Bound<'_, PyAny>) -> PyResult<()> { + Self::reify(slf)?; + let budget = ConversionBudget::full(); + for value in elem_seq.try_iter()? { + Self::append_with(slf, &value?, &budget)?; + } + Ok(()) + } } +/// The methods take their arguments as upb's do, positional only (but `add`'s keywords), with +/// upb's errors. #[pymethods] impl RepeatedComposite { /// As upb's: a new message of the fields of `kwargs`, appended; a positional argument is the @@ -912,29 +1185,37 @@ impl RepeatedComposite { /// As upb's: a new message, appended, then merged with `value` and returned, so that a message /// of another type leaves the new message (empty) in the container. + #[pyo3(signature = (*args, **kwargs))] fn append<'py>( slf: &Bound<'py, Self>, - value: &Bound<'py, PyAny>, + args: &Bound<'py, PyTuple>, + kwargs: Option<&Bound<'py, PyDict>>, ) -> PyResult> { - Self::append_with(slf, value, &ConversionBudget::full()) + let value = one_argument(Self::UPB_CLASS, "append", args, kwargs)?; + Self::append_with(slf, &value, &ConversionBudget::full()) } - /// As upb's: each value of the iterator of `elem_seq` appended as it comes, so a failure - /// leaves the values before it appended. One interrupt budget counts a step for each message - /// that the appends merge or copy, so that the extend of many messages lets another thread - /// take the GIL, and a signal interrupt it, as a copy does. - fn extend(slf: &Bound<'_, Self>, elem_seq: &Bound<'_, PyAny>) -> PyResult<()> { - Self::reify(slf)?; - let budget = ConversionBudget::full(); - for value in elem_seq.try_iter()? { - Self::append_with(slf, &value?, &budget)?; - } - Ok(()) + /// See [`RepeatedComposite::extend_values`]. + #[pyo3(signature = (*args, **kwargs))] + fn extend( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + let elem_seq = one_argument(Self::UPB_CLASS, "extend", args, kwargs)?; + Self::extend_values(slf, &elem_seq) } /// As upb's: the position of `key` in the container as it is, then the copy of `value`, /// which goes at the end of a container that is shorter by then. - fn insert(slf: &Bound<'_, Self>, key: isize, value: &Bound<'_, PyAny>) -> PyResult<()> { + #[pyo3(signature = (*args, **kwargs))] + fn insert( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + let (key, value) = insert_arguments(Self::UPB_CLASS, args, kwargs)?; + let value = &value; let py = slf.py(); Self::reify(slf)?; let (class, at) = { @@ -948,8 +1229,13 @@ impl RepeatedComposite { Ok(()) } - #[pyo3(signature = (key=-1))] - fn pop(slf: &Bound<'_, Self>, key: isize) -> PyResult> { + #[pyo3(signature = (*args, **kwargs))] + fn pop( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + let key = pop_argument(Self::UPB_CLASS, args, kwargs)?; let py = slf.py(); Self::reify(slf)?; let len = slf.borrow().items.len(); @@ -959,7 +1245,13 @@ impl RepeatedComposite { /// As upb's: the first message that equals `elem`, among as many as the container had, each /// of the container as it is when it is compared. - fn remove(slf: &Bound<'_, Self>, elem: &Bound<'_, PyAny>) -> PyResult<()> { + #[pyo3(signature = (*args, **kwargs))] + fn remove( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + let elem = &one_argument(Self::UPB_CLASS, "remove", args, kwargs)?; let py = slf.py(); Self::reify(slf)?; let len = slf.borrow().items.len(); @@ -1013,17 +1305,27 @@ impl RepeatedComposite { Ok(()) } - #[pyo3(name = "MergeFrom")] - fn merge_from(slf: &Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult<()> { - Self::extend(slf, other) + #[pyo3(name = "MergeFrom", signature = (*args, **kwargs))] + fn merge_from( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + let other = one_argument(Self::UPB_CLASS, "MergeFrom", args, kwargs)?; + Self::extend_values(slf, &other) } /// A copy of the container, of copies of its messages, held by no message. The messages are /// read under a short borrow and copied after it, of one interrupt budget, as a copy of a /// message copies them. - #[pyo3(signature = (memo=None))] - fn __deepcopy__(slf: &Bound<'_, Self>, memo: Option<&Bound<'_, PyAny>>) -> PyResult { - let _ = memo; + #[pyo3(signature = (*args, **kwargs))] + fn __deepcopy__( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult { + let _ = args; + no_keywords_of_positional(slf.py(), Self::UPB_CLASS, "__deepcopy__", kwargs)?; let py = slf.py(); let (class, items) = { let me = slf.borrow(); @@ -1372,6 +1674,16 @@ impl MapContainer { } } + /// The name of upb's class of the map, which its errors of the arguments of its methods name: + /// `ScalarMapContainer` (the runtime's `MapContainer`) or `MessageMapContainer`. + fn upb_class_name(slf: &Bound<'_, Self>) -> &'static str { + if slf.borrow().map.value == Kind::Message { + "MessageMapContainer" + } else { + "ScalarMapContainer" + } + } + /// The full name of the map field, which upb's errors name: of the message that holds the /// entry type (`spark.connect.SQL.named_arguments` of `spark.connect.SQL.NamedArgumentsEntry`). fn full_field_name(&self) -> String { @@ -1787,69 +2099,103 @@ impl MapContainer { } } - /// The keys, values and items, as the views of `collections.abc.Mapping` give them, as - /// upb's maps do. - fn keys<'py>(slf: &Bound<'py, Self>) -> PyResult> { - view(slf.as_any(), "KeysView") + /// The keys, values and items, as upb's maps give them: `Mapping`'s methods, of its views and + /// of the errors of their arguments. + #[pyo3(signature = (*args, **kwargs))] + fn keys( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + abc_method(slf, "Mapping", "keys", args, kwargs) } - fn values<'py>(slf: &Bound<'py, Self>) -> PyResult> { - view(slf.as_any(), "ValuesView") + #[pyo3(signature = (*args, **kwargs))] + fn values( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + abc_method(slf, "Mapping", "values", args, kwargs) } - fn items<'py>(slf: &Bound<'py, Self>) -> PyResult> { - view(slf.as_any(), "ItemsView") + #[pyo3(signature = (*args, **kwargs))] + fn items( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + abc_method(slf, "Mapping", "items", args, kwargs) } - #[pyo3(signature = (key, default=None))] + /// The value of `key`, or `default`, of upb's arguments and errors (see [`get_arguments`]). + #[pyo3(signature = (*args, **kwargs))] fn get( slf: &Bound<'_, Self>, - key: &Bound<'_, PyAny>, - default: Option>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { - let key = key_of(&Self::map_of(slf), key)?; - Ok(Self::lookup(slf, &key)?.unwrap_or_else(|| default.unwrap_or_else(|| slf.py().None()))) + let (key, default) = get_arguments(args, kwargs)?; + let key = key_of(&Self::map_of(slf), &key)?; + Ok(match Self::lookup(slf, &key)? { + Some(value) => value, + None => default.map_or_else(|| slf.py().None(), Bound::unbind), + }) } - /// The value of `key`, removed, as `MutableMapping.pop` of upb's maps gives it: `map[key]` - /// gives the default of the value type for a missing key, so `default` is never returned. - #[pyo3(signature = (key, default=None))] + /// As upb's, which is `MutableMapping.pop`: `map[key]` gives the default of the value type for + /// a missing key, so `default` is never returned. + #[pyo3(signature = (*args, **kwargs))] fn pop( slf: &Bound<'_, Self>, - key: &Bound<'_, PyAny>, - default: Option<&Bound<'_, PyAny>>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { - let _ = default; - let value = Self::get_or_insert(slf, key)?; - Self::__delitem__(slf, key)?; - Ok(value) + abc_method(slf, "MutableMapping", "pop", args, kwargs) } - fn popitem<'py>(slf: &Bound<'py, Self>) -> PyResult<(Bound<'py, PyAny>, Py)> { - let py = slf.py(); - let key = match slf.borrow().entries.iter().next() { - Some((key, _)) => key.clone(), - None => return Err(PyKeyError::new_err(())), - }; - let key = key.to_py(py)?.into_bound(py); - let value = Self::get_or_insert(slf, &key)?; - Self::__delitem__(slf, &key)?; - Ok((key, value)) + /// As upb's, which is `MutableMapping.popitem`: the first key of an iteration of the map. + #[pyo3(signature = (*args, **kwargs))] + fn popitem( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + abc_method(slf, "MutableMapping", "popitem", args, kwargs) } - #[pyo3(signature = (key, default=None))] + /// As upb's: of no keyword arguments, refused of a map of messages, and of a key and a value + /// that is not None, of the errors of `PyArg_UnpackTuple` of a map of scalars. + #[pyo3(signature = (*args, **kwargs))] fn setdefault( slf: &Bound<'_, Self>, - key: &Bound<'_, PyAny>, - default: Option<&Bound<'_, PyAny>>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { + no_keywords_of_positional(slf.py(), Self::upb_class_name(slf), "setdefault", kwargs)?; let map = Self::map_of(slf); if map.value == Kind::Message { return Err(PyNotImplementedError::new_err( "Set message map value directly is not supported.", )); } - let Some(default) = default.filter(|d| !d.is_none()) else { + match args.len() { + 0 => { + return Err(PyTypeError::new_err( + "setdefault expected at least 1 argument, got 0", + )) + } + 1 | 2 => {} + given => { + return Err(PyTypeError::new_err(format!( + "setdefault expected at most 2 arguments, got {given}" + ))) + } + } + let key = args.get_item(0)?; + let key = &key; + let default = args.get_item(1).ok(); + let Some(default) = default.as_ref().filter(|d| !d.is_none()) else { return Err(PyValueError::new_err( "The value for scalar map setdefault must be set.", )); @@ -1864,7 +2210,13 @@ impl MapContainer { Ok(default.clone().unbind()) } - fn clear(slf: &Bound<'_, Self>) -> PyResult<()> { + #[pyo3(signature = (*args, **kwargs))] + fn clear( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + no_arguments(Self::upb_class_name(slf), "clear", args, kwargs)?; Self::reify(slf)?; slf.borrow_mut().new_version(); // The entries are dropped once the map is no longer borrowed. @@ -1899,9 +2251,16 @@ impl MapContainer { /// copied into the message of the key. The entries of a map of other types are converted, with /// upb's errors. upb's map takes a new version first, so an iteration of the map in progress /// fails, even of an argument that it refuses. Each entry is a step of an interrupt budget. - #[pyo3(name = "MergeFrom")] - fn merge_from(slf: &Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult<()> { + /// The argument is upb's one positional argument, of the errors of `METH_O`, which come before + /// the new version. + #[pyo3(name = "MergeFrom", signature = (*args, **kwargs))] + fn merge_from( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { let py = slf.py(); + let other = &one_argument(Self::upb_class_name(slf), "MergeFrom", args, kwargs)?; Self::reify(slf)?; slf.borrow_mut().new_version(); if other.cast::().is_ok() { @@ -1911,11 +2270,18 @@ impl MapContainer { Self::update_from(py, slf, other, &field, &ConversionBudget::full()) } - /// The class of the entry messages of the map, as upb's `GetEntryClass`. - #[pyo3(name = "GetEntryClass")] - fn get_entry_class(&self, py: Python<'_>) -> PyResult> { - registered_class(py, &self.map.entry) - .ok_or_else(|| PyTypeError::new_err(format!("no class for {}", self.map.entry))) + /// The class of the entry messages of the map, as upb's `GetEntryClass`, of its errors of + /// arguments. + #[pyo3(name = "GetEntryClass", signature = (*args, **kwargs))] + fn get_entry_class( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + no_arguments(Self::upb_class_name(slf), "GetEntryClass", args, kwargs)?; + let entry = slf.borrow().map.entry.clone(); + registered_class(slf.py(), &entry) + .ok_or_else(|| PyTypeError::new_err(format!("no class for {entry}"))) } fn __richcmp__( @@ -2009,18 +2375,8 @@ impl MessageMapContainer { args: &Bound<'_, PyTuple>, kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { - if kwargs.is_some_and(|k| !k.is_empty()) { - return Err(PyTypeError::new_err( - "MessageMapContainer.get_or_create() takes no keyword arguments", - )); - } - if args.len() != 1 { - return Err(PyTypeError::new_err(format!( - "MessageMapContainer.get_or_create() takes exactly one argument ({} given)", - args.len() - ))); - } - MapContainer::get_or_insert(slf.as_super(), &args.get_item(0)?) + let key = one_argument("MessageMapContainer", "get_or_create", args, kwargs)?; + MapContainer::get_or_insert(slf.as_super(), &key) } } diff --git a/docs/compatibility.md b/docs/compatibility.md index 43820e3c..ee4ce5f7 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -1628,10 +1628,23 @@ protobuf 6's differ), the methods of protobuf's `Any` and `Timestamp` (a Timesta `unknown_fields.UnknownFieldSet` reads their unknown fields, which `text_format` prints with `print_unknown_fields=True` (see below). The differences: -- its types are of `pyspark.sql.connect.proto`, not `google.protobuf`: its - classes are of the metaclass `type`, not upb's `MessageMeta`, and can be subclassed, which upb - refuses, and its map of scalars is a `MapContainer`, where upb's is a `ScalarMapContainer`, of - which its map of messages, a `MessageMapContainer` as upb's, is a subclass; +- its types are of `pyspark.sql.connect.proto`, not `google.protobuf`: its classes are of the + metaclass `type`, not upb's `MessageMeta`, and can be subclassed, which upb refuses, and its map + of scalars is a `MapContainer`, where upb's is a `ScalarMapContainer`, of which its map of + messages, a `MessageMapContainer` as upb's, is a subclass. The methods of its containers take + their arguments as upb's do, positionally + (`RepeatedScalarContainer.append() takes no keyword arguments`), with upb's errors, of the + names of upb's classes of the maps too + (`ScalarMapContainer.MergeFrom() takes exactly one argument (0 given)`). A dunder method of a + map called directly with the wrong arguments (`map.__eq__()`) is an exception: its error is + that of CPython's slot wrapper of the native type (`expected 1 argument, got 0`), where upb's + `__eq__` of a map is `Mapping.__eq__`, a Python function. So is a keyword argument of + `setdefault`, `insert`, `pop` or `__deepcopy__` (upb's `METH_VARARGS` methods) called through a + bound method object (`f = map.setdefault; f(key="k")`): CPython's error of upb's names the + method only (`setdefault() takes no keyword arguments`), as of a bound builtin method, and the + runtime's method, which cannot tell that call from one of the map, names the class too. Before + Python 3.12, while a trace or a profile function is set (`coverage`, a debugger), CPython calls + each such method as a bound method, and the runtime's error then names the method only too; - the pool of its descriptors (`DESCRIPTOR.file.pool`) is its own, of the files of the protos of Spark Connect and of the plugins' files that the drop-in adds to it (see below), not `protobuf`'s default pool: it finds their descriptors as upb's does, but has no `Add`, @@ -1758,24 +1771,22 @@ and the rest of such a module is `protobuf`'s: `reflection.GeneratedProtocolMessageType` rather than with its builder, keeps `protobuf`'s classes, which take the package's messages only through bytes or dicts, whatever its file. -The other way, the messages of `pyspark.sql.connect.proto` take `protobuf`'s messages of their -types as they are, the `MergeFrom` of their maps takes the items of `protobuf`'s maps, as upb's -takes those of any mapping but a dict, and an `Any` of either packs and unpacks a message of -either. A message of the -runtime equals (`==`, on either side) a message of `protobuf`'s of its type of `protobuf`'s default -pool of the same contents, as upb's messages of its one class of a type do: the runtime parses the -bytes of `protobuf`'s message to compare it, as a field of the runtime takes it, so `in`, `remove` -and `==` of the containers find such messages too. A message of a type of another pool, of -another class for upb too, is not equal, and `CopyFrom`, `MergeFrom`, the constructors and the -containers refuse it with upb's errors of a message of another class, as they refuse an object +The other way, the messages of `pyspark.sql.connect.proto` take `protobuf`'s messages of their types +as they are, the `MergeFrom` of their maps takes the items of `protobuf`'s maps, as upb's takes +those of any mapping but a dict, and an `Any` of either packs and unpacks a message of either. A +message of the runtime equals (`==`, on either side) a message of `protobuf`'s of its type of +`protobuf`'s default pool of the same contents, as upb's messages of its one class of a type do: the +runtime parses the bytes of `protobuf`'s message to compare it, as a field of the runtime takes it, +so `in`, `remove` and `==` of the containers find such messages too. A message of a type of another +pool, of another class for upb too, is not equal, and `CopyFrom`, `MergeFrom`, the constructors and +the containers refuse it with upb's errors of a message of another class, as they refuse an object that only has the `DESCRIPTOR` and the `SerializeToString` of a message. As upb's, `==` of two -messages compares a float or a double field of its bits, so that a NaN equals a NaN of the same -bits and -0.0 is not 0.0 (nor the default of a field without presence), and their unknown fields -of their numbers, wire types and values, in the order of their tags, so that the order of fields -of other tags and the encoding of a varint do not count. `==` of the containers and maps compares -their values as Python's `==` does, as upb's `list` and `Mapping` comparisons do: a NaN value is -not equal to itself, and a map is equal to a map of other key or value types of equal keys and -values. +messages compares a float or a double field of its bits, so that a NaN equals a NaN of the same bits +and -0.0 is not 0.0 (nor the default of a field without presence), and their unknown fields of their +numbers, wire types and values, in the order of their tags, so that the order of fields of other +tags and the encoding of a varint do not count. `==` of the containers and maps compares their +values as Python's `==` does, as upb's `list` and `Mapping` comparisons do: a NaN value is not equal +to itself, and a map is equal to a map of other key or value types of equal keys and values. `google.protobuf.unknown_fields.UnknownFieldSet` is the drop-in's wrapper of `protobuf`'s class, once `pyspark.sql.connect.proto` is imported, which takes the messages of the package too, where diff --git a/python/tests/test_proto_rust_offline.py b/python/tests/test_proto_rust_offline.py index d4d9fa20..830520fd 100644 --- a/python/tests/test_proto_rust_offline.py +++ b/python/tests/test_proto_rust_offline.py @@ -1632,6 +1632,162 @@ def test_update_of_a_map_takes_its_arguments_as_upbs(upb): assert outcomes[0] == outcomes[1], label +_MAP_METHOD_CALLS = { + "MergeFrom()": lambda m: m.MergeFrom(), + "MergeFrom(a, b)": lambda m: m.MergeFrom([], []), + "MergeFrom(other=a)": lambda m: m.MergeFrom(other=[]), + "MergeFrom(a, x=1)": lambda m: m.MergeFrom([], x=1), + "clear(1)": lambda m: m.clear(1), + "clear(x=1)": lambda m: m.clear(x=1), + "clear(1, x=2)": lambda m: m.clear(1, x=2), + "GetEntryClass(1)": lambda m: m.GetEntryClass(1), + "GetEntryClass(x=1)": lambda m: m.GetEntryClass(x=1), + "get()": lambda m: m.get(), + "get(k, d, x)": lambda m: m.get("k", 1, 2), + "get(key=k)": lambda m: m.get(key="k"), + "get(zz, default=1)": lambda m: m.get("zz", default=1), + "get(x=1)": lambda m: m.get(x=1), + "get(default=1)": lambda m: m.get(default=1), + "get(k, x=1)": lambda m: m.get("k", x=1), + "get(k, key=z)": lambda m: m.get("k", key="z"), + "get(k, 1, default=2)": lambda m: m.get("k", 1, default=2), + "get(key=k, default=1, x=2)": lambda m: m.get(key="k", default=1, x=2), + "get(k, 2, 3, x=4)": lambda m: m.get("k", 2, 3, x=4), + "pop()": lambda m: m.pop(), + "pop(k, d, x)": lambda m: m.pop("k", 1, 2), + "pop(key=k)": lambda m: m.pop(key="k"), + "pop(k, key=z)": lambda m: m.pop("k", key="z"), + "pop(x=1)": lambda m: m.pop(x=1), + "popitem(1)": lambda m: m.popitem(1), + "popitem(x=1)": lambda m: m.popitem(x=1), + "popitem() of an empty map": lambda m: (m.clear(), m.popitem()), + "setdefault()": lambda m: m.setdefault(), + "setdefault(k, d, x)": lambda m: m.setdefault("k", "d", 2), + "setdefault(key=k)": lambda m: m.setdefault(key="k"), + "setdefault(q, default=d)": lambda m: m.setdefault("q", default="d"), + "setdefault(q, d)": lambda m: m.setdefault("q", "d"), + "keys(1)": lambda m: m.keys(1), + "keys(x=1)": lambda m: m.keys(x=1), + "values(1)": lambda m: m.values(1), + "items(x=1)": lambda m: m.items(x=1), + "sorted keys(), values(), items()": lambda m: ( + sorted(m.keys()), + len(m.values()), + [k for k, _ in m.items()], + ), + "update(a, b)": lambda m: m.update({}, {}), + "get_or_create()": lambda m: m.get_or_create(), + "get_or_create(a, b)": lambda m: m.get_or_create("a", "b"), + "get_or_create(key=k)": lambda m: m.get_or_create(key="k"), +} + + +_REPEATED_METHOD_CALLS = { + "append()": lambda c, v: c.append(), + "append(a, b)": lambda c, v: c.append(v, v), + "append(value=v)": lambda c, v: c.append(value=v), + "append(v, x=1)": lambda c, v: c.append(v, x=1), + "extend()": lambda c, v: c.extend(), + "extend(a, b)": lambda c, v: c.extend([], []), + "extend(elem_seq=[])": lambda c, v: c.extend(elem_seq=[]), + "insert()": lambda c, v: c.insert(), + "insert(0)": lambda c, v: c.insert(0), + "insert(0, v, 1)": lambda c, v: c.insert(0, v, 1), + "insert(key=0, value=v)": lambda c, v: c.insert(key=0, value=v), + "insert(a, v)": lambda c, v: c.insert("a", v), + "insert(2**70, v)": lambda c, v: c.insert(2**70, v), + "insert(-1, v)": lambda c, v: c.insert(-1, v), + "pop(0, 1)": lambda c, v: c.pop(0, 1), + "pop(key=0)": lambda c, v: c.pop(key=0), + "pop(a)": lambda c, v: c.pop("a"), + "pop(2**70)": lambda c, v: c.pop(2**70), + "pop()": lambda c, v: c.pop(), + "pop(0)": lambda c, v: c.pop(0), + "remove()": lambda c, v: c.remove(), + "remove(a, b)": lambda c, v: c.remove(v, v), + "remove(elem=v)": lambda c, v: c.remove(elem=v), + "MergeFrom()": lambda c, v: c.MergeFrom(), + "MergeFrom(a, b)": lambda c, v: c.MergeFrom([], []), + "MergeFrom(other=[])": lambda c, v: c.MergeFrom(other=[]), + "MergeFrom([v])": lambda c, v: c.MergeFrom([v]), + "reverse(1)": lambda c, v: c.reverse(1), + "clear(1)": lambda c, v: c.clear(1), + "sort(1)": lambda c, v: c.sort(1), + "__deepcopy__(1, 2)": lambda c, v: c.__deepcopy__(1, 2), + "__deepcopy__(memo={})": lambda c, v: c.__deepcopy__(memo={}), + "copy.deepcopy": lambda c, v: copy.deepcopy(c), +} + + +def test_methods_of_containers_raise_upbs_errors_of_their_arguments(upb): + # upb's maps are of its C types ScalarMapContainer and MessageMapContainer, whose own methods + # (MergeFrom, clear, GetEntryClass, setdefault, get_or_create) take positional arguments only, + # with CPython's errors of their C functions naming the class, and whose get parses its + # arguments with PyArg_ParseTupleAndKeywords; its pop, popitem, update, keys, values and items + # are those of collections.abc's MutableMapping and Mapping, in Python. upb's repeated + # containers take positional arguments only too, append, extend, remove and MergeFrom one, + # insert and pop of PyArg_ParseTuple's "nO" and "|n", with their errors. The runtime's methods + # had PyO3's arguments and errors: they took keyword arguments (MergeFrom(other=...), + # append(value=...), setdefault(key, default=...)), and the errors of the maps named + # MapContainer. + for kind, of in (("scalar", "Read.DataSource"), ("message", "SQL")): + for label, call in _MAP_METHOD_CALLS.items(): + if kind == "scalar" and label.startswith("get_or_create"): + continue + outcomes = [] + for new in (upb, ours): + if kind == "scalar": + message = new(f"spark.connect.{of}")(options={"k": "v"}) + container = message.options + else: + message = new(f"spark.connect.{of}")(args={"k": {"integer": 1}}) + container = message.args + outcome = _outcome(lambda: call(container)) + outcomes.append((outcome, message.SerializeToString(deterministic=True))) + assert outcomes[0] == outcomes[1], (kind, label) + for kind in ("scalar", "composite"): + for label, call in _REPEATED_METHOD_CALLS.items(): + outcomes = [] + for new in (upb, ours): + if kind == "scalar": + message = new("spark.connect.StatApproxQuantile")(probabilities=[1.0, 2.0]) + container, value = message.probabilities, 3.0 + else: + message = new("spark.connect.Expression.Literal.Array")( + elements=[{"integer": 1}, {"integer": 2}] + ) + container = message.elements + value = new("spark.connect.Expression.Literal")(integer=3) + outcome = _outcome(lambda: call(container, value)) + outcomes.append((outcome, message.SerializeToString())) + assert outcomes[0] == outcomes[1], (kind, label) + # A keyword argument of upb's methods of positional arguments (METH_VARARGS) is CPython's + # error of the method of its class when it is called of its object, and of the method only + # when CPython calls it as a bound method, as it calls each before Python 3.12 while a trace + # or a profile function is set (as coverage sets one): the runtime's errors follow. + calls = { + "setdefault": lambda new: new("spark.connect.Read.DataSource")().options.setdefault( + key="k" + ), + "insert": lambda new: new("spark.connect.StatApproxQuantile")().probabilities.insert( + key=0, value=1.0 + ), + "pop": lambda new: new("spark.connect.Expression.Literal.Array")().elements.pop(key=0), + "__deepcopy__": lambda new: new( + "spark.connect.StatApproxQuantile" + )().probabilities.__deepcopy__(memo={}), + } + previous = sys.getprofile() + for profile in (None, lambda *args: None): + sys.setprofile(profile) + try: + for label, call in calls.items(): + outcomes = [_outcome(lambda new=new: call(new)) for new in (upb, ours)] + assert outcomes[0] == outcomes[1], (label, profile) + finally: + sys.setprofile(previous) + + def _iterate_while(container, operation): """What a step of an iteration of `container` gives after `operation` ran between its first two steps: the outcome of the operation, then the keys of the rest of the iteration, or the