Add tokenize functions (#57)

* Add tokenize_json, tokenize_yaml

* Tweak error message. Expose Message and Position exports

* Tweak error message

Author: lovelydinosaur

docs/tokenized_errors.md

@@ -13,13 +13,13 @@ text = '''{
     "enable_auto_reload": "true"
 }'''
 
-value, messages = typesystem.validate_json(text, validator=Config)
-
-assert value is None
-for message in messages:
-    line_no = message.start_position.line_no
-    column_no = message.start_position.column_no
-    print(f"Error {message.text!r} at line {line_no}, column {column_no}.")
+try:
+    typesystem.validate_json(text, validator=Config)
+except (typesystem.ValidationError, typesystem.ParseError) as exc:
+    for message in exc.messages():
+        line_no = message.start_position.line_no
+        column_no = message.start_position.column_no
+        print(f"Error {message.text!r} at line {line_no}, column {column_no}.")
 # Error 'Must be a number.' at line 2, column 29.
 ```
 
@@ -29,6 +29,3 @@ The two functions for parsing content and providing positional error messages ar
 * `validate_yaml(text_or_bytes, validator)`
 
 In both cases `validator` may either be a `Schema` class, or a `Field` instance.
-
-Both functions return a two-tuple of `(value, messages)`. If the `messages` list
-is non-empty, then `value` will be `None`.

tests/tokenize/test_tokenize_json.py

@@ -1,8 +1,6 @@
-from json.decoder import JSONDecodeError
-
 import pytest
 
-from typesystem.tokenize.tokenize_json import tokenize_json
+from typesystem import ParseError, tokenize_json
 from typesystem.tokenize.tokens import DictToken, ListToken, ScalarToken
 
 
@@ -89,38 +87,62 @@ def test_tokenize_whitespace():
 
 
 def test_tokenize_parse_errors():
-    with pytest.raises(JSONDecodeError) as exc_info:
+    with pytest.raises(ParseError) as exc_info:
+        tokenize_json(b"")
+    exc = exc_info.value
+    message = exc.messages()[0]
+    assert message.text == "No content."
+    assert message.start_position.char_index == 0
+    assert (
+        repr(message)
+        == "Message(text='No content.', code='no_content', position=Position(line_no=1, column_no=1, char_index=0))"
+    )
+
+    with pytest.raises(ParseError) as exc_info:
         tokenize_json("{")
     exc = exc_info.value
-    assert exc.msg == "Expecting property name enclosed in double quotes"
-    assert exc.pos == 1
+    message = exc.messages()[0]
+    assert message.text == "Expecting property name enclosed in double quotes."
+    assert message.start_position.char_index == 1
 
-    with pytest.raises(JSONDecodeError) as exc_info:
+    with pytest.raises(ParseError) as exc_info:
         tokenize_json('{"a"')
     exc = exc_info.value
-    assert exc.msg == "Expecting ':' delimiter"
-    assert exc.pos == 4
+    message = exc.messages()[0]
+    assert message.text == "Expecting ':' delimiter."
+    assert message.start_position.char_index == 4
 
-    with pytest.raises(JSONDecodeError) as exc_info:
+    with pytest.raises(ParseError) as exc_info:
         tokenize_json('{"a":')
     exc = exc_info.value
-    assert exc.msg == "Expecting value"
-    assert exc.pos == 5
+    message = exc.messages()[0]
+    assert message.text == "Expecting value."
+    assert message.start_position.char_index == 5
 
-    with pytest.raises(JSONDecodeError) as exc_info:
+    with pytest.raises(ParseError) as exc_info:
         tokenize_json('{"a":1')
     exc = exc_info.value
-    assert exc.msg == "Expecting ',' delimiter"
-    assert exc.pos == 6
+    message = exc.messages()[0]
+    assert message.text == "Expecting ',' delimiter."
+    assert message.start_position.char_index == 6
 
-    with pytest.raises(JSONDecodeError) as exc_info:
+    with pytest.raises(ParseError) as exc_info:
         tokenize_json('{"a":1,1')
     exc = exc_info.value
-    assert exc.msg == "Expecting property name enclosed in double quotes"
-    assert exc.pos == 7
+    message = exc.messages()[0]
+    assert message.text == "Expecting property name enclosed in double quotes."
+    assert message.start_position.char_index == 7
 
-    with pytest.raises(JSONDecodeError) as exc_info:
+    with pytest.raises(ParseError) as exc_info:
         tokenize_json('{"a":1 "b"')
     exc = exc_info.value
-    assert exc.msg == "Expecting ',' delimiter"
-    assert exc.pos == 7
+    message = exc.messages()[0]
+    assert message.text == "Expecting ',' delimiter."
+    assert message.start_position.char_index == 7
+
+    with pytest.raises(ParseError) as exc_info:
+        tokenize_json('{"a" 123}')
+    exc = exc_info.value
+    message = exc.messages()[0]
+    assert message.text == "Expecting ':' delimiter."
+    assert message.start_position.char_index == 5

tests/tokenize/test_tokenize_yaml.py

@@ -1,4 +1,6 @@
-from typesystem.tokenize.tokenize_yaml import tokenize_yaml
+import pytest
+
+from typesystem import ParseError, tokenize_yaml
 from typesystem.tokenize.tokens import DictToken, ListToken, ScalarToken
 
 YAML_OBJECT = """
@@ -56,3 +58,19 @@ def test_tokenize_floats():
     token = tokenize_yaml(YAML_FLOATS)
     expected = ListToken([ScalarToken(100.0, 3, 7), ScalarToken(100.0, 11, 16)], 1, 17)
     assert token == expected
+
+
+def test_tokenize_parse_errors():
+    with pytest.raises(ParseError) as exc_info:
+        tokenize_yaml(b"")
+    exc = exc_info.value
+    message = exc.messages()[0]
+    assert message.text == "No content."
+    assert message.start_position.char_index == 0
+
+    with pytest.raises(ParseError) as exc_info:
+        tokenize_yaml('{"a" 1}')
+    exc = exc_info.value
+    message = exc.messages()[0]
+    assert message.text == "expected ',' or '}', but got '<scalar>'."
+    assert message.start_position.char_index == 5

tests/tokenize/test_validate_json.py

@@ -1,46 +1,24 @@
-from typesystem import Integer, Object, Schema
+import pytest
+
+from typesystem import Integer, Object, Schema, ValidationError
 from typesystem.base import Message, Position
 from typesystem.tokenize.tokenize_json import validate_json
 
 
 def test_validate_json():
-    value, messages = validate_json("")
-    assert value is None
-    assert messages == [
-        Message(
-            text="No content.",
-            code="no_content",
-            position=Position(line_no=1, column_no=1, char_index=0),
-        )
-    ]
-    assert (
-        repr(messages[0])
-        == "Message(text='No content.', code='no_content', position=Position(line_no=1, column_no=1, char_index=0))"
-    )
-
-    value, messages = validate_json('{"a": 123}')
-    assert value == {"a": 123}
-    assert messages == []
-
-    value, messages = validate_json(b'{"a": 123}')
-    assert value == {"a": 123}
-    assert messages == []
-
-    value, messages = validate_json('{"a" 123}')
-    assert value is None
-    assert messages == [
-        Message(
-            text="Expecting ':' delimiter.",
-            code="parse_error",
-            position=Position(line_no=1, column_no=6, char_index=5),
-        )
-    ]
+    validator = Object(properties=Integer())
+    text = '{\n    "a": "123",\n    "b": "456"}'
+    value = validate_json(text, validator=validator)
+    assert value == {"a": 123, "b": 456}
 
     validator = Object(properties=Integer())
     text = '{\n    "a": "123",\n    "b": "abc"}'
-    value, messages = validate_json(text, validator=validator)
-    assert value is None
-    assert messages == [
+
+    with pytest.raises(ValidationError) as exc_info:
+        validate_json(text, validator=validator)
+    exc = exc_info.value
+
+    assert exc.messages() == [
         Message(
             text="Must be a number.",
             code="type",
@@ -50,24 +28,19 @@ def test_validate_json():
         )
     ]
     assert (
-        repr(messages[0])
+        repr(exc.messages()[0])
         == "Message(text='Must be a number.', code='type', index=['b'], start_position=Position(line_no=3, column_no=10, char_index=27), end_position=Position(line_no=3, column_no=14, char_index=31))"
     )
 
-    validator = Object(properties=Integer())
-    text = '{\n    "a": "123",\n    "b": "456"}'
-    value, messages = validate_json(text, validator=validator)
-    assert value == {"a": 123, "b": 456}
-    assert messages == []
-
     class Validator(Schema):
         a = Integer()
         b = Integer()
 
     text = '{\n    "a": "123",\n    "b": "abc"}'
-    value, messages = validate_json(text, validator=Validator)
-    assert value is None
-    assert messages == [
+    with pytest.raises(ValidationError) as exc_info:
+        validate_json(text, validator=Validator)
+    exc = exc_info.value
+    assert exc.messages() == [
         Message(
             text="Must be a number.",
             code="type",
@@ -78,9 +51,10 @@ class Validator(Schema):
     ]
 
     text = '{"a": "123"}'
-    value, messages = validate_json(text, validator=Validator)
-    assert value is None
-    assert messages == [
+    with pytest.raises(ValidationError) as exc_info:
+        validate_json(text, validator=Validator)
+    exc = exc_info.value
+    assert exc.messages() == [
         Message(
             text="The field 'b' is required.",
             code="required",

tests/tokenize/test_validate_yaml.py

@@ -1,42 +1,23 @@
-from typesystem import Integer, Object, Schema
+import pytest
+
+from typesystem import Integer, Object, Schema, ValidationError, validate_yaml
 from typesystem.base import Message, Position
-from typesystem.tokenize.tokenize_yaml import validate_yaml
 
 
 def test_validate_yaml():
-    value, messages = validate_yaml("")
-    assert value is None
-    assert messages == [
-        Message(
-            text="No content.",
-            code="no_content",
-            position=Position(line_no=1, column_no=1, char_index=0),
-        )
-    ]
-
-    value, messages = validate_yaml("a: 123")
-    assert value == {"a": 123}
-    assert messages == []
-
-    value, messages = validate_yaml(b"a: 123")
-    assert value == {"a": 123}
-    assert messages == []
-
-    value, messages = validate_yaml('{"a" 1}')
-    assert value is None
-    assert messages == [
-        Message(
-            text="expected ',' or '}', but got '<scalar>'.",
-            code="parse_error",
-            position=Position(line_no=1, column_no=6, char_index=5),
-        )
-    ]
+    validator = Object(properties=Integer())
+    text = "a: 123\nb: 456\n"
+    value = validate_yaml(text, validator=validator)
+    assert value == {"a": 123, "b": 456}
 
     validator = Object(properties=Integer())
     text = "a: 123\nb: abc\n"
-    value, messages = validate_yaml(text, validator=validator)
-    assert value is None
-    assert messages == [
+
+    with pytest.raises(ValidationError) as exc_info:
+        validate_yaml(text, validator=validator)
+
+    exc = exc_info.value
+    assert exc.messages() == [
         Message(
             text="Must be a number.",
             code="type",
@@ -46,20 +27,17 @@ def test_validate_yaml():
         )
     ]
 
-    validator = Object(properties=Integer())
-    text = "a: 123\nb: 456\n"
-    value, messages = validate_yaml(text, validator=validator)
-    assert value == {"a": 123, "b": 456}
-    assert messages == []
-
     class Validator(Schema):
         a = Integer()
         b = Integer()
 
     text = "a: 123\nb: abc\n"
-    value, messages = validate_yaml(text, validator=Validator)
-    assert value is None
-    assert messages == [
+
+    with pytest.raises(ValidationError) as exc_info:
+        validate_yaml(text, validator=Validator)
+
+    exc = exc_info.value
+    assert exc.messages() == [
         Message(
             text="Must be a number.",
             code="type",
@@ -70,9 +48,10 @@ class Validator(Schema):
     ]
 
     text = "a: 123"
-    value, messages = validate_yaml(text, validator=Validator)
-    assert value is None
-    assert messages == [
+    with pytest.raises(ValidationError) as exc_info:
+        validate_yaml(text, validator=Validator)
+    exc = exc_info.value
+    assert exc.messages() == [
         Message(
             text="The field 'b' is required.",
             code="required",

typesystem/__init__.py

@@ -1,7 +1,7 @@
-from typesystem.base import ValidationError
+from typesystem.base import ParseError, ValidationError, Message, Position
 from typesystem.fields import (
-    Array,
     Any,
+    Array,
     Boolean,
     Choice,
     Date,
@@ -20,8 +20,9 @@
 from typesystem.forms import Jinja2Forms
 from typesystem.json_schema import from_json_schema, to_json_schema
 from typesystem.schemas import Reference, Schema, SchemaDefinitions
-from typesystem.tokenize.tokenize_json import validate_json
-from typesystem.tokenize.tokenize_yaml import validate_yaml
+from typesystem.tokenize.positional_validation import validate_with_positions
+from typesystem.tokenize.tokenize_json import tokenize_json, validate_json
+from typesystem.tokenize.tokenize_yaml import tokenize_yaml, validate_yaml
 
 __version__ = "0.1.12"
 __all__ = [
@@ -39,15 +40,25 @@
     "Number",
     "Object",
     "Reference",
-    "Schema",
     "String",
     "Text",
     "Time",
     "Union",
-    "ValidationError",
+    # Schemas
+    "Schema",
     "SchemaDefinitions",
+    # Exceptions
+    "ParseError",
+    "ValidationError",
+    "Message",
+    "Position",
+    # JSON Schema
     "from_json_schema",
     "to_json_schema",
+    # Positional error marking
+    "tokenize_json",
+    "tokenize_yaml",
     "validate_json",
     "validate_yaml",
+    "validate_with_positions",
 ]