feature: redo python api

* matchers can be combined using (`~`, `|`, `&`)
* depth matcher can be used
* a bigger refactor of the matching logic

Author: shenek

src/lib.rs

@@ -6,37 +6,104 @@ use std::sync::{Arc, Mutex};
 use streamson_lib::{error, handler, matcher, Collector};
 
 create_exception!(streamson, StreamsonError, exceptions::ValueError);
+create_exception!(streamson, MatcherUsed, exceptions::RuntimeError);
 
 impl From<error::General> for StreamsonError {
     fn from(_gerror: error::General) -> Self {
         Self
     }
 }
 
+/// Python wrapper around matchers
+#[pyclass]
+#[derive(Debug)]
+pub struct RustMatcher {
+    inner: Option<matcher::Combinator>,
+}
+
+#[pymethods]
+impl RustMatcher {
+    /// Create a new instance of simple matcher
+    ///
+    /// # Arguments
+    /// * `path` - path to match
+    /// * `max_depth` - max depth
+    #[staticmethod]
+    pub fn simple(path: String) -> PyResult<Self> {
+        Ok(Self {
+            inner: Some(matcher::Combinator::new(matcher::Simple::new(path))),
+        })
+    }
+
+    /// Create a new instance of depth matcher
+    ///
+    /// # Arguments
+    /// * `min_depth` - min depth
+    /// * `max_depth` - max depth (Optional)
+    #[staticmethod]
+    pub fn depth(min_depth: usize, max_depth: Option<usize>) -> PyResult<Self> {
+        Ok(Self {
+            inner: Some(matcher::Combinator::new(matcher::Depth::new(
+                min_depth, max_depth,
+            ))),
+        })
+    }
+
+    pub fn inv(&mut self) -> PyResult<Self> {
+        if let Some(inner) = self.inner.take() {
+            Ok(Self {
+                inner: Some(!inner),
+            })
+        } else {
+            Err(MatcherUsed.into())
+        }
+    }
+
+    pub fn any(&mut self, right: &mut RustMatcher) -> PyResult<Self> {
+        if let (Some(left), Some(right)) = (self.inner.take(), right.inner.take()) {
+            Ok(Self {
+                inner: Some(left | right),
+            })
+        } else {
+            Err(MatcherUsed.into())
+        }
+    }
+
+    pub fn all(&mut self, right: &mut RustMatcher) -> PyResult<Self> {
+        if let (Some(left), Some(right)) = (self.inner.take(), right.inner.take()) {
+            Ok(Self {
+                inner: Some(left & right),
+            })
+        } else {
+            Err(MatcherUsed.into())
+        }
+    }
+
+    #[getter]
+    pub fn _used(&self) -> bool {
+        self.inner.is_none()
+    }
+}
+
 /// Low level Python wrapper for Simple matcher and Buffer handler
 #[pyclass]
-pub struct SimpleStreamson {
+pub struct Streamson {
     collector: Collector,
     handler: Arc<Mutex<handler::Buffer>>,
 }
 
 #[pymethods]
-impl SimpleStreamson {
-    /// Create a new instance of SimpleStreamson
+impl Streamson {
+    /// Create a new instance of Streamson
     ///
     /// # Arguments
     /// * `matches` - a list of valid simple matches (e.g. `{"users"}`, `[]{"name"}`, `[0]{}`)
     #[new]
-    pub fn new(matches: Vec<String>) -> Self {
+    pub fn new(matcher: &mut RustMatcher) -> PyResult<Self> {
         let handler = Arc::new(Mutex::new(handler::Buffer::new()));
-        let mut collector = Collector::new();
-        for path_match in matches {
-            collector = collector.add_matcher(
-                Box::new(matcher::Simple::new(path_match)),
-                &[handler.clone()],
-            );
-        }
-        Self { collector, handler }
+        let matcher = matcher.inner.take().ok_or(MatcherUsed)?;
+        let collector = Collector::new().add_matcher(Box::new(matcher), &[handler.clone()]);
+        Ok(Self { collector, handler })
     }
 
     /// Feeds Streamson processor with data
@@ -66,7 +133,8 @@ impl SimpleStreamson {
 /// This module is a python module implemented in Rust.
 #[pymodule]
 fn streamson(_py: Python, m: &PyModule) -> PyResult<()> {
-    m.add_class::<SimpleStreamson>()?;
+    m.add_class::<Streamson>()?;
+    m.add_class::<RustMatcher>()?;
 
     Ok(())
 }

streamson/__init__.py

@@ -2,19 +2,47 @@
 import typing
 from array import array
 
-from streamson.streamson import SimpleStreamson as _SimpleStreamson
+from streamson.streamson import RustMatcher as _RustMatcher
+from streamson.streamson import Streamson as _Streamson
+
+
+class Matcher:
+    def __init__(self, rust_matcher: _RustMatcher):
+        self.inner = rust_matcher
+
+    def __invert__(self):
+        self.inner = self.inner.inv()
+        return self
+
+    def __or__(self, other):
+        self.inner = self.inner.any(other.inner)
+        return self
+
+    def __and__(self, other):
+        self.inner = self.inner.all(other.inner)
+        return self
+
+
+class DepthMatcher(Matcher):
+    def __init__(self, min_depth: int, max_depth: typing.Optional[int] = None):
+        super().__init__(_RustMatcher.depth(min_depth, max_depth))
+
+
+class SimpleMatcher(Matcher):
+    def __init__(self, path: str):
+        super().__init__(_RustMatcher.simple(path))
 
 
 def extract_iter(
-    input_gen: typing.Generator[bytes, None, None], simple_matches: typing.List[str],
+    input_gen: typing.Generator[bytes, None, None], matcher: Matcher,
 ) -> typing.Generator[typing.Tuple[str, typing.Any], None, None]:
     """ Extracts json specified by givem list of simple matches
     :param: input_gen - input generator
     :param: simple_matches - matches to check
 
     :returns: (string, data) generator
     """
-    streamson = _SimpleStreamson(simple_matches)
+    streamson = _Streamson(matcher.inner)
     for item in input_gen:
         streamson.feed(item)
         res = streamson.pop()

tests/test_basic.py

@@ -0,0 +1,80 @@
+import pytest
+
+import streamson
+
+DATA = [b'{"users": ["john","carl","bob"]}']
+
+
+def test_simple():
+    matcher = streamson.SimpleMatcher('{"users"}[]')
+    extracted = streamson.extract_iter((e for e in DATA), matcher)
+    assert next(extracted) == ('{"users"}[0]', "john")
+    assert next(extracted) == ('{"users"}[1]', "carl")
+    assert next(extracted) == ('{"users"}[2]', "bob")
+
+    with pytest.raises(StopIteration):
+        next(extracted)
+
+
+def test_depth():
+    matcher = streamson.DepthMatcher(1)
+    extracted = streamson.extract_iter((e for e in DATA), matcher)
+    assert next(extracted) == ('{"users"}[0]', "john")
+    assert next(extracted) == ('{"users"}[1]', "carl")
+    assert next(extracted) == ('{"users"}[2]', "bob")
+    assert next(extracted) == ('{"users"}', ["john", "carl", "bob"])
+
+    with pytest.raises(StopIteration):
+        next(extracted)
+
+    matcher = streamson.DepthMatcher(0, 1)
+    extracted = streamson.extract_iter((e for e in DATA), matcher)
+    assert next(extracted) == ('{"users"}', ["john", "carl", "bob"])
+    assert next(extracted) == ("", {"users": ["john", "carl", "bob"]})
+
+    with pytest.raises(StopIteration):
+        next(extracted)
+
+
+def test_invert():
+    matcher = ~streamson.DepthMatcher(2)
+    extracted = streamson.extract_iter((e for e in DATA), matcher)
+    assert next(extracted) == ('{"users"}', ["john", "carl", "bob"])
+    assert next(extracted) == ("", {"users": ["john", "carl", "bob"]})
+
+
+def test_all():
+    matcher = streamson.SimpleMatcher('{"users"}[]') & streamson.SimpleMatcher("{}[1]")
+
+    extracted = streamson.extract_iter((e for e in DATA), matcher)
+    assert next(extracted) == ('{"users"}[1]', "carl")
+
+    with pytest.raises(StopIteration):
+        next(extracted)
+
+
+def test_any():
+    matcher = streamson.DepthMatcher(2, 2) | streamson.SimpleMatcher('{"users"}')
+
+    extracted = streamson.extract_iter((e for e in DATA), matcher)
+    assert next(extracted) == ('{"users"}[0]', "john")
+    assert next(extracted) == ('{"users"}[1]', "carl")
+    assert next(extracted) == ('{"users"}[2]', "bob")
+    assert next(extracted) == ('{"users"}', ["john", "carl", "bob"])
+
+    with pytest.raises(StopIteration):
+        next(extracted)
+
+
+def test_complex():
+    matcher = (streamson.DepthMatcher(2, 2) | streamson.SimpleMatcher('{"users"}')) & ~streamson.SimpleMatcher(
+        '{"users"}[0]'
+    )
+
+    extracted = streamson.extract_iter((e for e in DATA), matcher)
+    assert next(extracted) == ('{"users"}[1]', "carl")
+    assert next(extracted) == ('{"users"}[2]', "bob")
+    assert next(extracted) == ('{"users"}', ["john", "carl", "bob"])
+
+    with pytest.raises(StopIteration):
+        next(extracted)