diff --git a/crates/pyspark-rs/src/pbmsg/containers.rs b/crates/pyspark-rs/src/pbmsg/containers.rs index 78bf43df..41c54403 100644 --- a/crates/pyspark-rs/src/pbmsg/containers.rs +++ b/crates/pyspark-rs/src/pbmsg/containers.rs @@ -269,13 +269,216 @@ fn not_in_container() -> PyErr { PyValueError::new_err("remove(x): x not in container") } -/// A container of `value`'s items, by `collections.abc` `name` (such as `KeysView`) of `value`. -fn view<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult> { - value - .py() - .import("collections.abc")? - .getattr(name)? - .call1((value,)) +/// upb's errors of the arguments of a method `method` of a container of upb's class `class` +/// (`ScalarMapContainer`, `MessageMapContainer`, `RepeatedScalarContainer`, ...) that takes none +/// (`METH_NOARGS`), as CPython checks them: a keyword argument first. +fn no_arguments( + class: &str, + method: &str, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult<()> { + no_keywords(class, method, kwargs)?; + if !args.is_empty() { + return Err(PyTypeError::new_err(format!( + "{class}.{method}() takes no arguments ({} given)", + args.len() + ))); + } + Ok(()) +} + +/// upb's errors of the arguments of a method of a container that takes exactly one (`METH_O`), as +/// [`no_arguments`] gives them: the argument. +fn one_argument<'py>( + class: &str, + method: &str, + args: &Bound<'py, PyTuple>, + kwargs: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + no_keywords(class, method, kwargs)?; + if args.len() != 1 { + return Err(PyTypeError::new_err(format!( + "{class}.{method}() takes exactly one argument ({} given)", + args.len() + ))); + } + args.get_item(0) +} + +/// upb's error of a keyword argument of a method of a container that takes one or none +/// (`METH_O`, `METH_NOARGS`), which names the method of its class however it is called. +fn no_keywords(class: &str, method: &str, kwargs: Option<&Bound<'_, PyDict>>) -> PyResult<()> { + if kwargs.is_some_and(|k| !k.is_empty()) { + return Err(PyTypeError::new_err(format!( + "{class}.{method}() takes no keyword arguments" + ))); + } + Ok(()) +} + +/// upb's error of a keyword argument of a method of a container that takes positional arguments +/// (`METH_VARARGS`: `setdefault`, `insert`, `pop`, `__deepcopy__`), which CPython words by the +/// way that the method is called: called of its object (`map.setdefault(key="k")`), it names the +/// method of its class (`ScalarMapContainer.setdefault() takes no keyword arguments`), and called +/// as a bound method, it names the method only (`setdefault() takes no keyword arguments`). The +/// runtime's methods cannot tell the two apart, so the error names the method of its class, but +/// where CPython binds the method of every call: before 3.12, while a trace or a profile function +/// is set on the thread (as `coverage` and debuggers set one), CPython's `trace_call_function` +/// calls a method of a native type as a bound method, so that a profiler sees it. +fn no_keywords_of_positional( + py: Python<'_>, + class: &str, + method: &str, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult<()> { + if kwargs.is_none_or(|k| k.is_empty()) { + return Ok(()); + } + let bound = py.version_info() < (3, 12) && { + let sys = py.import("sys")?; + !sys.call_method0("gettrace")?.is_none() || !sys.call_method0("getprofile")?.is_none() + }; + Err(PyTypeError::new_err(if bound { + format!("{method}() takes no keyword arguments") + } else { + format!("{class}.{method}() takes no keyword arguments") + })) +} + +/// The `key` and `default` of upb's `get` of a map, which parses them as CPython's +/// `PyArg_ParseTupleAndKeywords(args, kwargs, "O|O", {"key", "default"})` does, with its errors +/// (of a "function", as the format names none): too many arguments, a missing `key`, an argument +/// given by name and position, then a keyword of no argument. +fn get_arguments<'py>( + args: &Bound<'py, PyTuple>, + kwargs: Option<&Bound<'py, PyDict>>, +) -> PyResult<(Bound<'py, PyAny>, Option>)> { + const NAMES: [&str; 2] = ["key", "default"]; + let nargs = args.len(); + let nkwargs = kwargs.map_or(0, |k| k.len()); + if nargs + nkwargs > NAMES.len() { + return Err(PyTypeError::new_err(format!( + "function takes at most {} {}arguments ({} given)", + NAMES.len(), + if nargs == 0 { "keyword " } else { "" }, + nargs + nkwargs + ))); + } + let keyword = |name: &str| -> PyResult>> { + match kwargs { + Some(kwargs) => kwargs.get_item(name), + None => Ok(None), + } + }; + let mut values: [Option>; 2] = [None, None]; + for (i, name) in NAMES.iter().enumerate() { + values[i] = if i < nargs { + Some(args.get_item(i)?) + } else { + keyword(name)? + }; + if values[i].is_none() && i == 0 { + return Err(PyTypeError::new_err( + "function missing required argument 'key' (pos 1)", + )); + } + } + for (i, name) in NAMES.iter().enumerate().take(nargs) { + if keyword(name)?.is_some() { + return Err(PyTypeError::new_err(format!( + "argument for function given by name ('{name}') and position ({})", + i + 1 + ))); + } + } + for key in kwargs.iter().flat_map(|k| k.keys()) { + if !key + .extract::() + .is_ok_and(|k| NAMES.contains(&k.as_str())) + { + return Err(PyTypeError::new_err(format!( + "'{}' is an invalid keyword argument for this function", + key.str()? + ))); + } + } + let [key, default] = values; + Ok((key.expect("the key, which is required"), default)) +} + +/// `arg` as an index of upb's `PyArg_ParseTuple` "n" format: of its `__index__`, with CPython's +/// errors ("'str' object cannot be interpreted as an integer", "Python int too large to convert to +/// C ssize_t"). +fn ssize_t_of(arg: &Bound<'_, PyAny>) -> PyResult { + let py = arg.py(); + // SAFETY: the pointers are of live objects; `PyNumber_Index` gives a new reference, owned by + // the `Bound` that wraps it, or null with an exception set. + unsafe { + let index = ffi::PyNumber_Index(arg.as_ptr()); + if index.is_null() { + return Err(PyErr::fetch(py)); + } + let index = Bound::from_owned_ptr(py, index); + let value = ffi::PyLong_AsSsize_t(index.as_ptr()); + match PyErr::take(py) { + Some(error) if value == -1 => Err(error), + _ => Ok(value), + } + } +} + +/// The index and the value of upb's `insert` of a repeated field of `class`, as it takes them of +/// `PyArg_ParseTuple(args, "nO")`: of no keyword argument (`METH_VARARGS`), exactly two, the index +/// of [`ssize_t_of`]. +fn insert_arguments<'py>( + class: &str, + args: &Bound<'py, PyTuple>, + kwargs: Option<&Bound<'py, PyDict>>, +) -> PyResult<(isize, Bound<'py, PyAny>)> { + no_keywords_of_positional(args.py(), class, "insert", kwargs)?; + if args.len() != 2 { + return Err(PyTypeError::new_err(format!( + "function takes exactly 2 arguments ({} given)", + args.len() + ))); + } + Ok((ssize_t_of(&args.get_item(0)?)?, args.get_item(1)?)) +} + +/// The index of upb's `pop` of a repeated field of `class`, as it takes it of +/// `PyArg_ParseTuple(args, "|n")`: of no keyword argument, at most one, -1 of none. +fn pop_argument( + class: &str, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult { + no_keywords_of_positional(args.py(), class, "pop", kwargs)?; + match args.len() { + 0 => Ok(-1), + 1 => ssize_t_of(&args.get_item(0)?), + given => Err(PyTypeError::new_err(format!( + "function takes at most 1 argument ({given} given)" + ))), + } +} + +/// The method `name` of `collections.abc`'s class `abc` (`Mapping`, `MutableMapping`) called of +/// `map` and of `args` and `kwargs`, as upb's maps have it: so its result, and the errors of its +/// arguments, are Python's (`MutableMapping.pop() missing 1 required positional argument: 'key'`). +fn abc_method( + map: &Bound<'_, MapContainer>, + abc: &str, + name: &str, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, +) -> PyResult> { + let py = map.py(); + let method = py.import("collections.abc")?.getattr(abc)?.getattr(name)?; + let mut all = Vec::with_capacity(args.len() + 1); + all.push(map.as_any().clone()); + all.extend(args.iter()); + Ok(method.call(PyTuple::new(py, all)?, kwargs)?.unbind()) } /// The key and value of `pair`, as `for key, value in pairs` unpacks it. @@ -590,20 +793,15 @@ impl RepeatedScalar { } } -#[pymethods] impl RepeatedScalar { - fn append(slf: &Bound<'_, Self>, value: &Bound<'_, PyAny>) -> PyResult<()> { - Self::reify(slf)?; - let v = Self::field_of(slf).convert(value)?; - slf.borrow_mut().items.push(v); - Ok(()) - } + /// The name of upb's class of the container, which its errors of arguments name. + const UPB_CLASS: &'static str = "RepeatedScalarContainer"; /// As upb's: the values of `elem_seq` (of `PySequence_Fast`, so those of an iterator to its /// end first), each converted and appended in turn. A failure, of the iteration too, leaves /// the container of the length that it had: upb resizes it, so a position that Python code /// removed meanwhile holds the default. - fn extend(slf: &Bound<'_, Self>, elem_seq: &Bound<'_, PyAny>) -> PyResult<()> { + fn extend_values(slf: &Bound<'_, Self>, elem_seq: &Bound<'_, PyAny>) -> PyResult<()> { Self::reify(slf)?; let field = Self::field_of(slf); let start = slf.borrow().items.len(); @@ -615,11 +813,46 @@ impl RepeatedScalar { } extended } +} + +/// The methods take their arguments as upb's do, positional only, with upb's errors. +#[pymethods] +impl RepeatedScalar { + #[pyo3(signature = (*args, **kwargs))] + fn append( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + let value = &one_argument(Self::UPB_CLASS, "append", args, kwargs)?; + Self::reify(slf)?; + let v = Self::field_of(slf).convert(value)?; + slf.borrow_mut().items.push(v); + Ok(()) + } + + /// See [`RepeatedScalar::extend_values`]. + #[pyo3(signature = (*args, **kwargs))] + fn extend( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + let elem_seq = one_argument(Self::UPB_CLASS, "extend", args, kwargs)?; + Self::extend_values(slf, &elem_seq) + } /// As upb's: the position of `key` in the container as it is, then the value converted. A /// conversion that shortened the container leaves the position past its end (where upb's /// insert writes out of its bounds), so the value goes at its end. - fn insert(slf: &Bound<'_, Self>, key: isize, value: &Bound<'_, PyAny>) -> PyResult<()> { + #[pyo3(signature = (*args, **kwargs))] + fn insert( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + let (key, value) = insert_arguments(Self::UPB_CLASS, args, kwargs)?; + let value = &value; Self::reify(slf)?; let at = insert_position(key, slf.borrow().items.len()); let v = Self::field_of(slf).convert(value)?; @@ -629,8 +862,13 @@ impl RepeatedScalar { Ok(()) } - #[pyo3(signature = (key=-1))] - fn pop(slf: &Bound<'_, Self>, key: isize) -> PyResult> { + #[pyo3(signature = (*args, **kwargs))] + fn pop( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + let key = pop_argument(Self::UPB_CLASS, args, kwargs)?; let py = slf.py(); Self::reify(slf)?; let len = slf.borrow().items.len(); @@ -642,7 +880,13 @@ impl RepeatedScalar { /// As upb's: the first item that equals `elem`, as Python compares them, among as many items /// as the container had, each of the container as it is when it is compared (an `__eq__` may /// change it). - fn remove(slf: &Bound<'_, Self>, elem: &Bound<'_, PyAny>) -> PyResult<()> { + #[pyo3(signature = (*args, **kwargs))] + fn remove( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + let elem = &one_argument(Self::UPB_CLASS, "remove", args, kwargs)?; let py = slf.py(); Self::reify(slf)?; let len = slf.borrow().items.len(); @@ -692,16 +936,27 @@ impl RepeatedScalar { Ok(()) } - #[pyo3(name = "MergeFrom")] - fn merge_from(slf: &Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult<()> { - Self::extend(slf, other) + #[pyo3(name = "MergeFrom", signature = (*args, **kwargs))] + fn merge_from( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + let other = one_argument(Self::UPB_CLASS, "MergeFrom", args, kwargs)?; + Self::extend_values(slf, &other) } - /// A copy of the container, held by no message, as upb's `copy.deepcopy` gives one. - #[pyo3(signature = (memo=None))] - fn __deepcopy__(&self, memo: Option<&Bound<'_, PyAny>>) -> Self { - let _ = memo; - self.copy() + /// A copy of the container, held by no message, as upb's `copy.deepcopy` gives one: of any + /// positional arguments (the memo) and of no keyword argument, as upb's `METH_VARARGS`. + #[pyo3(signature = (*args, **kwargs))] + fn __deepcopy__( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult { + let _ = args; + no_keywords_of_positional(slf.py(), Self::UPB_CLASS, "__deepcopy__", kwargs)?; + Ok(slf.borrow().copy()) } fn __reduce__(&self, py: Python<'_>) -> PyResult> { @@ -884,8 +1139,26 @@ impl RepeatedComposite { PyMessage::merge_from_with(&child, value, budget)?; Ok(child) } + + /// The name of upb's class of the container, which its errors of arguments name. + const UPB_CLASS: &'static str = "RepeatedCompositeContainer"; + + /// As upb's: each value of the iterator of `elem_seq` appended as it comes, so a failure + /// leaves the values before it appended. One interrupt budget counts a step for each message + /// that the appends merge or copy, so that the extend of many messages lets another thread + /// take the GIL, and a signal interrupt it, as a copy does. + fn extend_values(slf: &Bound<'_, Self>, elem_seq: &Bound<'_, PyAny>) -> PyResult<()> { + Self::reify(slf)?; + let budget = ConversionBudget::full(); + for value in elem_seq.try_iter()? { + Self::append_with(slf, &value?, &budget)?; + } + Ok(()) + } } +/// The methods take their arguments as upb's do, positional only (but `add`'s keywords), with +/// upb's errors. #[pymethods] impl RepeatedComposite { /// As upb's: a new message of the fields of `kwargs`, appended; a positional argument is the @@ -912,29 +1185,37 @@ impl RepeatedComposite { /// As upb's: a new message, appended, then merged with `value` and returned, so that a message /// of another type leaves the new message (empty) in the container. + #[pyo3(signature = (*args, **kwargs))] fn append<'py>( slf: &Bound<'py, Self>, - value: &Bound<'py, PyAny>, + args: &Bound<'py, PyTuple>, + kwargs: Option<&Bound<'py, PyDict>>, ) -> PyResult> { - Self::append_with(slf, value, &ConversionBudget::full()) + let value = one_argument(Self::UPB_CLASS, "append", args, kwargs)?; + Self::append_with(slf, &value, &ConversionBudget::full()) } - /// As upb's: each value of the iterator of `elem_seq` appended as it comes, so a failure - /// leaves the values before it appended. One interrupt budget counts a step for each message - /// that the appends merge or copy, so that the extend of many messages lets another thread - /// take the GIL, and a signal interrupt it, as a copy does. - fn extend(slf: &Bound<'_, Self>, elem_seq: &Bound<'_, PyAny>) -> PyResult<()> { - Self::reify(slf)?; - let budget = ConversionBudget::full(); - for value in elem_seq.try_iter()? { - Self::append_with(slf, &value?, &budget)?; - } - Ok(()) + /// See [`RepeatedComposite::extend_values`]. + #[pyo3(signature = (*args, **kwargs))] + fn extend( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + let elem_seq = one_argument(Self::UPB_CLASS, "extend", args, kwargs)?; + Self::extend_values(slf, &elem_seq) } /// As upb's: the position of `key` in the container as it is, then the copy of `value`, /// which goes at the end of a container that is shorter by then. - fn insert(slf: &Bound<'_, Self>, key: isize, value: &Bound<'_, PyAny>) -> PyResult<()> { + #[pyo3(signature = (*args, **kwargs))] + fn insert( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + let (key, value) = insert_arguments(Self::UPB_CLASS, args, kwargs)?; + let value = &value; let py = slf.py(); Self::reify(slf)?; let (class, at) = { @@ -948,8 +1229,13 @@ impl RepeatedComposite { Ok(()) } - #[pyo3(signature = (key=-1))] - fn pop(slf: &Bound<'_, Self>, key: isize) -> PyResult> { + #[pyo3(signature = (*args, **kwargs))] + fn pop( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + let key = pop_argument(Self::UPB_CLASS, args, kwargs)?; let py = slf.py(); Self::reify(slf)?; let len = slf.borrow().items.len(); @@ -959,7 +1245,13 @@ impl RepeatedComposite { /// As upb's: the first message that equals `elem`, among as many as the container had, each /// of the container as it is when it is compared. - fn remove(slf: &Bound<'_, Self>, elem: &Bound<'_, PyAny>) -> PyResult<()> { + #[pyo3(signature = (*args, **kwargs))] + fn remove( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + let elem = &one_argument(Self::UPB_CLASS, "remove", args, kwargs)?; let py = slf.py(); Self::reify(slf)?; let len = slf.borrow().items.len(); @@ -1013,17 +1305,27 @@ impl RepeatedComposite { Ok(()) } - #[pyo3(name = "MergeFrom")] - fn merge_from(slf: &Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult<()> { - Self::extend(slf, other) + #[pyo3(name = "MergeFrom", signature = (*args, **kwargs))] + fn merge_from( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + let other = one_argument(Self::UPB_CLASS, "MergeFrom", args, kwargs)?; + Self::extend_values(slf, &other) } /// A copy of the container, of copies of its messages, held by no message. The messages are /// read under a short borrow and copied after it, of one interrupt budget, as a copy of a /// message copies them. - #[pyo3(signature = (memo=None))] - fn __deepcopy__(slf: &Bound<'_, Self>, memo: Option<&Bound<'_, PyAny>>) -> PyResult { - let _ = memo; + #[pyo3(signature = (*args, **kwargs))] + fn __deepcopy__( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult { + let _ = args; + no_keywords_of_positional(slf.py(), Self::UPB_CLASS, "__deepcopy__", kwargs)?; let py = slf.py(); let (class, items) = { let me = slf.borrow(); @@ -1372,6 +1674,16 @@ impl MapContainer { } } + /// The name of upb's class of the map, which its errors of the arguments of its methods name: + /// `ScalarMapContainer` (the runtime's `MapContainer`) or `MessageMapContainer`. + fn upb_class_name(slf: &Bound<'_, Self>) -> &'static str { + if slf.borrow().map.value == Kind::Message { + "MessageMapContainer" + } else { + "ScalarMapContainer" + } + } + /// The full name of the map field, which upb's errors name: of the message that holds the /// entry type (`spark.connect.SQL.named_arguments` of `spark.connect.SQL.NamedArgumentsEntry`). fn full_field_name(&self) -> String { @@ -1787,69 +2099,103 @@ impl MapContainer { } } - /// The keys, values and items, as the views of `collections.abc.Mapping` give them, as - /// upb's maps do. - fn keys<'py>(slf: &Bound<'py, Self>) -> PyResult> { - view(slf.as_any(), "KeysView") + /// The keys, values and items, as upb's maps give them: `Mapping`'s methods, of its views and + /// of the errors of their arguments. + #[pyo3(signature = (*args, **kwargs))] + fn keys( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + abc_method(slf, "Mapping", "keys", args, kwargs) } - fn values<'py>(slf: &Bound<'py, Self>) -> PyResult> { - view(slf.as_any(), "ValuesView") + #[pyo3(signature = (*args, **kwargs))] + fn values( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + abc_method(slf, "Mapping", "values", args, kwargs) } - fn items<'py>(slf: &Bound<'py, Self>) -> PyResult> { - view(slf.as_any(), "ItemsView") + #[pyo3(signature = (*args, **kwargs))] + fn items( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + abc_method(slf, "Mapping", "items", args, kwargs) } - #[pyo3(signature = (key, default=None))] + /// The value of `key`, or `default`, of upb's arguments and errors (see [`get_arguments`]). + #[pyo3(signature = (*args, **kwargs))] fn get( slf: &Bound<'_, Self>, - key: &Bound<'_, PyAny>, - default: Option>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { - let key = key_of(&Self::map_of(slf), key)?; - Ok(Self::lookup(slf, &key)?.unwrap_or_else(|| default.unwrap_or_else(|| slf.py().None()))) + let (key, default) = get_arguments(args, kwargs)?; + let key = key_of(&Self::map_of(slf), &key)?; + Ok(match Self::lookup(slf, &key)? { + Some(value) => value, + None => default.map_or_else(|| slf.py().None(), Bound::unbind), + }) } - /// The value of `key`, removed, as `MutableMapping.pop` of upb's maps gives it: `map[key]` - /// gives the default of the value type for a missing key, so `default` is never returned. - #[pyo3(signature = (key, default=None))] + /// As upb's, which is `MutableMapping.pop`: `map[key]` gives the default of the value type for + /// a missing key, so `default` is never returned. + #[pyo3(signature = (*args, **kwargs))] fn pop( slf: &Bound<'_, Self>, - key: &Bound<'_, PyAny>, - default: Option<&Bound<'_, PyAny>>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { - let _ = default; - let value = Self::get_or_insert(slf, key)?; - Self::__delitem__(slf, key)?; - Ok(value) + abc_method(slf, "MutableMapping", "pop", args, kwargs) } - fn popitem<'py>(slf: &Bound<'py, Self>) -> PyResult<(Bound<'py, PyAny>, Py)> { - let py = slf.py(); - let key = match slf.borrow().entries.iter().next() { - Some((key, _)) => key.clone(), - None => return Err(PyKeyError::new_err(())), - }; - let key = key.to_py(py)?.into_bound(py); - let value = Self::get_or_insert(slf, &key)?; - Self::__delitem__(slf, &key)?; - Ok((key, value)) + /// As upb's, which is `MutableMapping.popitem`: the first key of an iteration of the map. + #[pyo3(signature = (*args, **kwargs))] + fn popitem( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + abc_method(slf, "MutableMapping", "popitem", args, kwargs) } - #[pyo3(signature = (key, default=None))] + /// As upb's: of no keyword arguments, refused of a map of messages, and of a key and a value + /// that is not None, of the errors of `PyArg_UnpackTuple` of a map of scalars. + #[pyo3(signature = (*args, **kwargs))] fn setdefault( slf: &Bound<'_, Self>, - key: &Bound<'_, PyAny>, - default: Option<&Bound<'_, PyAny>>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { + no_keywords_of_positional(slf.py(), Self::upb_class_name(slf), "setdefault", kwargs)?; let map = Self::map_of(slf); if map.value == Kind::Message { return Err(PyNotImplementedError::new_err( "Set message map value directly is not supported.", )); } - let Some(default) = default.filter(|d| !d.is_none()) else { + match args.len() { + 0 => { + return Err(PyTypeError::new_err( + "setdefault expected at least 1 argument, got 0", + )) + } + 1 | 2 => {} + given => { + return Err(PyTypeError::new_err(format!( + "setdefault expected at most 2 arguments, got {given}" + ))) + } + } + let key = args.get_item(0)?; + let key = &key; + let default = args.get_item(1).ok(); + let Some(default) = default.as_ref().filter(|d| !d.is_none()) else { return Err(PyValueError::new_err( "The value for scalar map setdefault must be set.", )); @@ -1864,7 +2210,13 @@ impl MapContainer { Ok(default.clone().unbind()) } - fn clear(slf: &Bound<'_, Self>) -> PyResult<()> { + #[pyo3(signature = (*args, **kwargs))] + fn clear( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { + no_arguments(Self::upb_class_name(slf), "clear", args, kwargs)?; Self::reify(slf)?; slf.borrow_mut().new_version(); // The entries are dropped once the map is no longer borrowed. @@ -1899,9 +2251,16 @@ impl MapContainer { /// copied into the message of the key. The entries of a map of other types are converted, with /// upb's errors. upb's map takes a new version first, so an iteration of the map in progress /// fails, even of an argument that it refuses. Each entry is a step of an interrupt budget. - #[pyo3(name = "MergeFrom")] - fn merge_from(slf: &Bound<'_, Self>, other: &Bound<'_, PyAny>) -> PyResult<()> { + /// The argument is upb's one positional argument, of the errors of `METH_O`, which come before + /// the new version. + #[pyo3(name = "MergeFrom", signature = (*args, **kwargs))] + fn merge_from( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult<()> { let py = slf.py(); + let other = &one_argument(Self::upb_class_name(slf), "MergeFrom", args, kwargs)?; Self::reify(slf)?; slf.borrow_mut().new_version(); if other.cast::().is_ok() { @@ -1911,11 +2270,18 @@ impl MapContainer { Self::update_from(py, slf, other, &field, &ConversionBudget::full()) } - /// The class of the entry messages of the map, as upb's `GetEntryClass`. - #[pyo3(name = "GetEntryClass")] - fn get_entry_class(&self, py: Python<'_>) -> PyResult> { - registered_class(py, &self.map.entry) - .ok_or_else(|| PyTypeError::new_err(format!("no class for {}", self.map.entry))) + /// The class of the entry messages of the map, as upb's `GetEntryClass`, of its errors of + /// arguments. + #[pyo3(name = "GetEntryClass", signature = (*args, **kwargs))] + fn get_entry_class( + slf: &Bound<'_, Self>, + args: &Bound<'_, PyTuple>, + kwargs: Option<&Bound<'_, PyDict>>, + ) -> PyResult> { + no_arguments(Self::upb_class_name(slf), "GetEntryClass", args, kwargs)?; + let entry = slf.borrow().map.entry.clone(); + registered_class(slf.py(), &entry) + .ok_or_else(|| PyTypeError::new_err(format!("no class for {entry}"))) } fn __richcmp__( @@ -2009,18 +2375,8 @@ impl MessageMapContainer { args: &Bound<'_, PyTuple>, kwargs: Option<&Bound<'_, PyDict>>, ) -> PyResult> { - if kwargs.is_some_and(|k| !k.is_empty()) { - return Err(PyTypeError::new_err( - "MessageMapContainer.get_or_create() takes no keyword arguments", - )); - } - if args.len() != 1 { - return Err(PyTypeError::new_err(format!( - "MessageMapContainer.get_or_create() takes exactly one argument ({} given)", - args.len() - ))); - } - MapContainer::get_or_insert(slf.as_super(), &args.get_item(0)?) + let key = one_argument("MessageMapContainer", "get_or_create", args, kwargs)?; + MapContainer::get_or_insert(slf.as_super(), &key) } } diff --git a/docs/compatibility.md b/docs/compatibility.md index 43820e3c..ee4ce5f7 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -1628,10 +1628,23 @@ protobuf 6's differ), the methods of protobuf's `Any` and `Timestamp` (a Timesta `unknown_fields.UnknownFieldSet` reads their unknown fields, which `text_format` prints with `print_unknown_fields=True` (see below). The differences: -- its types are of `pyspark.sql.connect.proto`, not `google.protobuf`: its - classes are of the metaclass `type`, not upb's `MessageMeta`, and can be subclassed, which upb - refuses, and its map of scalars is a `MapContainer`, where upb's is a `ScalarMapContainer`, of - which its map of messages, a `MessageMapContainer` as upb's, is a subclass; +- its types are of `pyspark.sql.connect.proto`, not `google.protobuf`: its classes are of the + metaclass `type`, not upb's `MessageMeta`, and can be subclassed, which upb refuses, and its map + of scalars is a `MapContainer`, where upb's is a `ScalarMapContainer`, of which its map of + messages, a `MessageMapContainer` as upb's, is a subclass. The methods of its containers take + their arguments as upb's do, positionally + (`RepeatedScalarContainer.append() takes no keyword arguments`), with upb's errors, of the + names of upb's classes of the maps too + (`ScalarMapContainer.MergeFrom() takes exactly one argument (0 given)`). A dunder method of a + map called directly with the wrong arguments (`map.__eq__()`) is an exception: its error is + that of CPython's slot wrapper of the native type (`expected 1 argument, got 0`), where upb's + `__eq__` of a map is `Mapping.__eq__`, a Python function. So is a keyword argument of + `setdefault`, `insert`, `pop` or `__deepcopy__` (upb's `METH_VARARGS` methods) called through a + bound method object (`f = map.setdefault; f(key="k")`): CPython's error of upb's names the + method only (`setdefault() takes no keyword arguments`), as of a bound builtin method, and the + runtime's method, which cannot tell that call from one of the map, names the class too. Before + Python 3.12, while a trace or a profile function is set (`coverage`, a debugger), CPython calls + each such method as a bound method, and the runtime's error then names the method only too; - the pool of its descriptors (`DESCRIPTOR.file.pool`) is its own, of the files of the protos of Spark Connect and of the plugins' files that the drop-in adds to it (see below), not `protobuf`'s default pool: it finds their descriptors as upb's does, but has no `Add`, @@ -1758,24 +1771,22 @@ and the rest of such a module is `protobuf`'s: `reflection.GeneratedProtocolMessageType` rather than with its builder, keeps `protobuf`'s classes, which take the package's messages only through bytes or dicts, whatever its file. -The other way, the messages of `pyspark.sql.connect.proto` take `protobuf`'s messages of their -types as they are, the `MergeFrom` of their maps takes the items of `protobuf`'s maps, as upb's -takes those of any mapping but a dict, and an `Any` of either packs and unpacks a message of -either. A message of the -runtime equals (`==`, on either side) a message of `protobuf`'s of its type of `protobuf`'s default -pool of the same contents, as upb's messages of its one class of a type do: the runtime parses the -bytes of `protobuf`'s message to compare it, as a field of the runtime takes it, so `in`, `remove` -and `==` of the containers find such messages too. A message of a type of another pool, of -another class for upb too, is not equal, and `CopyFrom`, `MergeFrom`, the constructors and the -containers refuse it with upb's errors of a message of another class, as they refuse an object +The other way, the messages of `pyspark.sql.connect.proto` take `protobuf`'s messages of their types +as they are, the `MergeFrom` of their maps takes the items of `protobuf`'s maps, as upb's takes +those of any mapping but a dict, and an `Any` of either packs and unpacks a message of either. A +message of the runtime equals (`==`, on either side) a message of `protobuf`'s of its type of +`protobuf`'s default pool of the same contents, as upb's messages of its one class of a type do: the +runtime parses the bytes of `protobuf`'s message to compare it, as a field of the runtime takes it, +so `in`, `remove` and `==` of the containers find such messages too. A message of a type of another +pool, of another class for upb too, is not equal, and `CopyFrom`, `MergeFrom`, the constructors and +the containers refuse it with upb's errors of a message of another class, as they refuse an object that only has the `DESCRIPTOR` and the `SerializeToString` of a message. As upb's, `==` of two -messages compares a float or a double field of its bits, so that a NaN equals a NaN of the same -bits and -0.0 is not 0.0 (nor the default of a field without presence), and their unknown fields -of their numbers, wire types and values, in the order of their tags, so that the order of fields -of other tags and the encoding of a varint do not count. `==` of the containers and maps compares -their values as Python's `==` does, as upb's `list` and `Mapping` comparisons do: a NaN value is -not equal to itself, and a map is equal to a map of other key or value types of equal keys and -values. +messages compares a float or a double field of its bits, so that a NaN equals a NaN of the same bits +and -0.0 is not 0.0 (nor the default of a field without presence), and their unknown fields of their +numbers, wire types and values, in the order of their tags, so that the order of fields of other +tags and the encoding of a varint do not count. `==` of the containers and maps compares their +values as Python's `==` does, as upb's `list` and `Mapping` comparisons do: a NaN value is not equal +to itself, and a map is equal to a map of other key or value types of equal keys and values. `google.protobuf.unknown_fields.UnknownFieldSet` is the drop-in's wrapper of `protobuf`'s class, once `pyspark.sql.connect.proto` is imported, which takes the messages of the package too, where diff --git a/python/tests/test_proto_rust_offline.py b/python/tests/test_proto_rust_offline.py index d4d9fa20..830520fd 100644 --- a/python/tests/test_proto_rust_offline.py +++ b/python/tests/test_proto_rust_offline.py @@ -1632,6 +1632,162 @@ def test_update_of_a_map_takes_its_arguments_as_upbs(upb): assert outcomes[0] == outcomes[1], label +_MAP_METHOD_CALLS = { + "MergeFrom()": lambda m: m.MergeFrom(), + "MergeFrom(a, b)": lambda m: m.MergeFrom([], []), + "MergeFrom(other=a)": lambda m: m.MergeFrom(other=[]), + "MergeFrom(a, x=1)": lambda m: m.MergeFrom([], x=1), + "clear(1)": lambda m: m.clear(1), + "clear(x=1)": lambda m: m.clear(x=1), + "clear(1, x=2)": lambda m: m.clear(1, x=2), + "GetEntryClass(1)": lambda m: m.GetEntryClass(1), + "GetEntryClass(x=1)": lambda m: m.GetEntryClass(x=1), + "get()": lambda m: m.get(), + "get(k, d, x)": lambda m: m.get("k", 1, 2), + "get(key=k)": lambda m: m.get(key="k"), + "get(zz, default=1)": lambda m: m.get("zz", default=1), + "get(x=1)": lambda m: m.get(x=1), + "get(default=1)": lambda m: m.get(default=1), + "get(k, x=1)": lambda m: m.get("k", x=1), + "get(k, key=z)": lambda m: m.get("k", key="z"), + "get(k, 1, default=2)": lambda m: m.get("k", 1, default=2), + "get(key=k, default=1, x=2)": lambda m: m.get(key="k", default=1, x=2), + "get(k, 2, 3, x=4)": lambda m: m.get("k", 2, 3, x=4), + "pop()": lambda m: m.pop(), + "pop(k, d, x)": lambda m: m.pop("k", 1, 2), + "pop(key=k)": lambda m: m.pop(key="k"), + "pop(k, key=z)": lambda m: m.pop("k", key="z"), + "pop(x=1)": lambda m: m.pop(x=1), + "popitem(1)": lambda m: m.popitem(1), + "popitem(x=1)": lambda m: m.popitem(x=1), + "popitem() of an empty map": lambda m: (m.clear(), m.popitem()), + "setdefault()": lambda m: m.setdefault(), + "setdefault(k, d, x)": lambda m: m.setdefault("k", "d", 2), + "setdefault(key=k)": lambda m: m.setdefault(key="k"), + "setdefault(q, default=d)": lambda m: m.setdefault("q", default="d"), + "setdefault(q, d)": lambda m: m.setdefault("q", "d"), + "keys(1)": lambda m: m.keys(1), + "keys(x=1)": lambda m: m.keys(x=1), + "values(1)": lambda m: m.values(1), + "items(x=1)": lambda m: m.items(x=1), + "sorted keys(), values(), items()": lambda m: ( + sorted(m.keys()), + len(m.values()), + [k for k, _ in m.items()], + ), + "update(a, b)": lambda m: m.update({}, {}), + "get_or_create()": lambda m: m.get_or_create(), + "get_or_create(a, b)": lambda m: m.get_or_create("a", "b"), + "get_or_create(key=k)": lambda m: m.get_or_create(key="k"), +} + + +_REPEATED_METHOD_CALLS = { + "append()": lambda c, v: c.append(), + "append(a, b)": lambda c, v: c.append(v, v), + "append(value=v)": lambda c, v: c.append(value=v), + "append(v, x=1)": lambda c, v: c.append(v, x=1), + "extend()": lambda c, v: c.extend(), + "extend(a, b)": lambda c, v: c.extend([], []), + "extend(elem_seq=[])": lambda c, v: c.extend(elem_seq=[]), + "insert()": lambda c, v: c.insert(), + "insert(0)": lambda c, v: c.insert(0), + "insert(0, v, 1)": lambda c, v: c.insert(0, v, 1), + "insert(key=0, value=v)": lambda c, v: c.insert(key=0, value=v), + "insert(a, v)": lambda c, v: c.insert("a", v), + "insert(2**70, v)": lambda c, v: c.insert(2**70, v), + "insert(-1, v)": lambda c, v: c.insert(-1, v), + "pop(0, 1)": lambda c, v: c.pop(0, 1), + "pop(key=0)": lambda c, v: c.pop(key=0), + "pop(a)": lambda c, v: c.pop("a"), + "pop(2**70)": lambda c, v: c.pop(2**70), + "pop()": lambda c, v: c.pop(), + "pop(0)": lambda c, v: c.pop(0), + "remove()": lambda c, v: c.remove(), + "remove(a, b)": lambda c, v: c.remove(v, v), + "remove(elem=v)": lambda c, v: c.remove(elem=v), + "MergeFrom()": lambda c, v: c.MergeFrom(), + "MergeFrom(a, b)": lambda c, v: c.MergeFrom([], []), + "MergeFrom(other=[])": lambda c, v: c.MergeFrom(other=[]), + "MergeFrom([v])": lambda c, v: c.MergeFrom([v]), + "reverse(1)": lambda c, v: c.reverse(1), + "clear(1)": lambda c, v: c.clear(1), + "sort(1)": lambda c, v: c.sort(1), + "__deepcopy__(1, 2)": lambda c, v: c.__deepcopy__(1, 2), + "__deepcopy__(memo={})": lambda c, v: c.__deepcopy__(memo={}), + "copy.deepcopy": lambda c, v: copy.deepcopy(c), +} + + +def test_methods_of_containers_raise_upbs_errors_of_their_arguments(upb): + # upb's maps are of its C types ScalarMapContainer and MessageMapContainer, whose own methods + # (MergeFrom, clear, GetEntryClass, setdefault, get_or_create) take positional arguments only, + # with CPython's errors of their C functions naming the class, and whose get parses its + # arguments with PyArg_ParseTupleAndKeywords; its pop, popitem, update, keys, values and items + # are those of collections.abc's MutableMapping and Mapping, in Python. upb's repeated + # containers take positional arguments only too, append, extend, remove and MergeFrom one, + # insert and pop of PyArg_ParseTuple's "nO" and "|n", with their errors. The runtime's methods + # had PyO3's arguments and errors: they took keyword arguments (MergeFrom(other=...), + # append(value=...), setdefault(key, default=...)), and the errors of the maps named + # MapContainer. + for kind, of in (("scalar", "Read.DataSource"), ("message", "SQL")): + for label, call in _MAP_METHOD_CALLS.items(): + if kind == "scalar" and label.startswith("get_or_create"): + continue + outcomes = [] + for new in (upb, ours): + if kind == "scalar": + message = new(f"spark.connect.{of}")(options={"k": "v"}) + container = message.options + else: + message = new(f"spark.connect.{of}")(args={"k": {"integer": 1}}) + container = message.args + outcome = _outcome(lambda: call(container)) + outcomes.append((outcome, message.SerializeToString(deterministic=True))) + assert outcomes[0] == outcomes[1], (kind, label) + for kind in ("scalar", "composite"): + for label, call in _REPEATED_METHOD_CALLS.items(): + outcomes = [] + for new in (upb, ours): + if kind == "scalar": + message = new("spark.connect.StatApproxQuantile")(probabilities=[1.0, 2.0]) + container, value = message.probabilities, 3.0 + else: + message = new("spark.connect.Expression.Literal.Array")( + elements=[{"integer": 1}, {"integer": 2}] + ) + container = message.elements + value = new("spark.connect.Expression.Literal")(integer=3) + outcome = _outcome(lambda: call(container, value)) + outcomes.append((outcome, message.SerializeToString())) + assert outcomes[0] == outcomes[1], (kind, label) + # A keyword argument of upb's methods of positional arguments (METH_VARARGS) is CPython's + # error of the method of its class when it is called of its object, and of the method only + # when CPython calls it as a bound method, as it calls each before Python 3.12 while a trace + # or a profile function is set (as coverage sets one): the runtime's errors follow. + calls = { + "setdefault": lambda new: new("spark.connect.Read.DataSource")().options.setdefault( + key="k" + ), + "insert": lambda new: new("spark.connect.StatApproxQuantile")().probabilities.insert( + key=0, value=1.0 + ), + "pop": lambda new: new("spark.connect.Expression.Literal.Array")().elements.pop(key=0), + "__deepcopy__": lambda new: new( + "spark.connect.StatApproxQuantile" + )().probabilities.__deepcopy__(memo={}), + } + previous = sys.getprofile() + for profile in (None, lambda *args: None): + sys.setprofile(profile) + try: + for label, call in calls.items(): + outcomes = [_outcome(lambda new=new: call(new)) for new in (upb, ours)] + assert outcomes[0] == outcomes[1], (label, profile) + finally: + sys.setprofile(previous) + + def _iterate_while(container, operation): """What a step of an iteration of `container` gives after `operation` ran between its first two steps: the outcome of the operation, then the keys of the rest of the iteration, or the