Skip to content

Commit ee19e69

Browse files
alessiostallaalessiostalla
committed
Align transformers with Kolasu in the following aspects:
- `with_child` API (order of parameters) - drop null children - one node can generate multiple nodes Also add a partial implementation of assert_asts_are_equal (not considering references as those aren't yet supported in reflection)
1 parent 4c996a8 commit ee19e69

10 files changed

Lines changed: 174 additions & 44 deletions

File tree

‎pylasu/model/model.py‎

Lines changed: 5 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
import enum
21
import inspect
32
from abc import ABC, abstractmethod, ABCMeta
43
from dataclasses import Field, MISSING, dataclass, field
54
from typing import Optional, Callable, List
65

76
from .position import Position, Source
7+
from .reflection import Multiplicity, PropertyDescription
88
from ..reflection import getannotations
99
from ..reflection.reflection import is_sequence_type, get_type_arguments
1010

@@ -73,22 +73,6 @@ def is_internal_property_or_method(value):
7373
return isinstance(value, internal_property) or isinstance(value, InternalField) or isinstance(value, Callable)
7474

7575

76-
class Multiplicity(enum.Enum):
77-
OPTIONAL = 0
78-
SINGULAR = 1
79-
MANY = 2
80-
81-
82-
@dataclass
83-
class PropertyDescriptor:
84-
name: str
85-
provides_nodes: bool
86-
multiplicity: Multiplicity = Multiplicity.SINGULAR
87-
88-
def multiple(self):
89-
return self.multiplicity == Multiplicity.MANY
90-
91-
9276
def provides_nodes(decl_type):
9377
return isinstance(decl_type, type) and issubclass(decl_type, Node)
9478

@@ -123,11 +107,11 @@ def _direct_node_properties(cls, cl, known_property_names):
123107
else:
124108
is_child_property = provides_nodes(decl_type)
125109
known_property_names.add(name)
126-
yield PropertyDescriptor(name, is_child_property, multiplicity)
110+
yield PropertyDescription(name, is_child_property, multiplicity)
127111
for name in dir(cl):
128112
if name not in known_property_names and cls.is_node_property(name):
129113
known_property_names.add(name)
130-
yield PropertyDescriptor(name, False)
114+
yield PropertyDescription(name, False)
131115

132116
def is_node_property(cls, name):
133117
return not name.startswith('_') \
@@ -178,10 +162,8 @@ def source(self) -> Optional[Source]:
178162

179163
@internal_property
180164
def properties(self):
181-
return ((name, getattr(self, name)) for name in dir(self)
182-
if not name.startswith('_')
183-
and name not in self.__internal_properties__
184-
and name not in [n for n, v in inspect.getmembers(type(self), is_internal_property_or_method)])
165+
return (PropertyDescription(p.name, p.provides_nodes, p.multiplicity, getattr(self, p.name))
166+
for p in self.__class__.node_properties)
185167

186168
@internal_property
187169
def _fields(self):

‎pylasu/model/reflection.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
import enum
2+
from dataclasses import dataclass
3+
4+
5+
class Multiplicity(enum.Enum):
6+
OPTIONAL = 0
7+
SINGULAR = 1
8+
MANY = 2
9+
10+
11+
@dataclass
12+
class PropertyDescription:
13+
name: str
14+
provides_nodes: bool
15+
multiplicity: Multiplicity = Multiplicity.SINGULAR
16+
value: object = None
17+
18+
@property
19+
def multiple(self):
20+
return self.multiplicity == Multiplicity.MANY

‎pylasu/testing/testing.py‎

Lines changed: 64 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,67 @@
1+
import unittest
2+
13
from pylasu.model import Node
24

35

4-
def assert_asts_are_equal(expected: Node, actual: Node, context: str = "<root>", consider_position: bool = False):
5-
raise NotImplementedError("TODO implement this. Transformers tests don't use this yet.")
6+
def assert_asts_are_equal(
7+
case: unittest.TestCase,
8+
expected: Node, actual: Node,
9+
context: str = "<root>", consider_position: bool = False
10+
):
11+
if expected.node_type != actual.node_type:
12+
case.fail(f"{context}: expected node of type {expected.node_type}, "
13+
f"but found {actual.node_type}")
14+
if consider_position:
15+
case.assertEqual(expected.position, actual.position, f"{context}.position")
16+
for expected_property in expected.properties:
17+
try:
18+
actual_property = next(filter(lambda p: p.name == expected_property.name, actual.properties))
19+
except StopIteration:
20+
case.fail(f"No property {expected_property.name} found at {context}")
21+
actual_prop_value = actual_property.value
22+
expected_prop_value = expected_property.value
23+
if expected_property.provides_nodes:
24+
if expected_property.multiple:
25+
assert_multi_properties_are_equal(
26+
case, expected_property, expected_prop_value, actual_prop_value, context, consider_position)
27+
else:
28+
assert_single_properties_are_equal(case, expected_property, expected_prop_value, actual_prop_value,
29+
context, consider_position)
30+
# TODO not yet supported elif expected_property.property_type == PropertyType.REFERENCE:
31+
else:
32+
case.assertEqual(
33+
expected_prop_value, actual_prop_value,
34+
f"{context}, comparing property {expected_property.name} of {expected.node_type}")
35+
36+
37+
def assert_single_properties_are_equal(case, expected_property, expected_prop_value, actual_prop_value, context,
38+
consider_position):
39+
if expected_prop_value is None and actual_prop_value is not None:
40+
case.assertEqual(expected_prop_value, actual_prop_value,
41+
f"{context}.{expected_property.name}")
42+
elif expected_prop_value is not None and actual_prop_value is None:
43+
case.assertEqual(expected_prop_value, actual_prop_value,
44+
f"{context}.{expected_property.name}")
45+
elif expected_prop_value is None and actual_prop_value is None:
46+
# that is ok
47+
pass
48+
else:
49+
case.assertIsInstance(actual_prop_value, Node)
50+
assert_asts_are_equal(
51+
case, expected_prop_value, actual_prop_value,
52+
context=f"{context}.{expected_property.name}",
53+
consider_position=consider_position)
54+
55+
56+
def assert_multi_properties_are_equal(case, expected_property, expected_prop_value, actual_prop_value, context,
57+
consider_position):
58+
# TODO IgnoreChildren
59+
case.assertEquals(actual_prop_value is None, expected_prop_value is None,
60+
f"{context}.{expected_property.name} nullness")
61+
if actual_prop_value is not None and expected_prop_value is not None:
62+
case.assertEquals(len(actual_prop_value), len(expected_prop_value),
63+
f"{context}.{expected_property.name} length")
64+
for expected_it, actual_it, i in \
65+
zip(expected_prop_value, actual_prop_value, range(len(expected_prop_value))):
66+
assert_asts_are_equal(case, expected_it, actual_it, f"{context}[{i}]",
67+
consider_position=consider_position)

‎pylasu/transformation/generic_nodes.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,4 @@
66
@dataclass
77
class GenericNode(Node):
88
"""A generic AST node. We use it to represent parts of a source tree that we don't know how to translate yet."""
9-
parent: Node
9+
parent: Node = None

‎pylasu/transformation/transformation.py‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
from pylasu.model import Node, Origin
77
from pylasu.model.errors import GenericErrorNode
8-
from pylasu.model.model import PropertyDescriptor
8+
from pylasu.model.reflection import PropertyDescription
99
from pylasu.transformation.generic_nodes import GenericNode
1010
from pylasu.validation import Issue, IssueSeverity
1111

@@ -36,8 +36,8 @@ class NodeFactory(Generic[Source, Output]):
3636

3737
def with_child(
3838
self,
39-
getter: Union[Callable[[Source], Optional[Any]], PropertyRef],
4039
setter: Union[Callable[[Target, Optional[Child]], None], PropertyRef],
40+
getter: Union[Callable[[Source], Optional[Any]], PropertyRef],
4141
name: Optional[str] = None,
4242
target_type: Optional[type] = None
4343
) -> "NodeFactory[Source, Output]":
@@ -140,10 +140,12 @@ def process_child(self, source, node, pd, factory):
140140
def as_origin(self, source: Any) -> Optional[Origin]:
141141
return source if isinstance(source, Origin) else None
142142

143-
def set_child(self, child_node_factory: ChildNodeFactory, source: Any, node: Node, pd: PropertyDescriptor):
143+
def set_child(self, child_node_factory: ChildNodeFactory, source: Any, node: Node, pd: PropertyDescription):
144144
src = child_node_factory.get(self.get_source(node, source))
145-
if pd.multiple():
146-
child = [self.transform(it, node) for it in src or [] if it is not None]
145+
if pd.multiple:
146+
child = []
147+
for child_src in src:
148+
child.extend(self.transform_into_nodes(child_src, node))
147149
else:
148150
child = self.transform(src, node)
149151
try:
@@ -191,13 +193,16 @@ def register_identity_transformation(self, node_class: Type[Target]):
191193
self.register_node_factory(node_class, lambda node: node)
192194

193195

194-
def get_node_constructor_wrapper(decorated_function):
195-
def ensure_list(obj):
196-
if isinstance(obj, list):
197-
return obj
198-
else:
199-
return [obj]
196+
def ensure_list(obj):
197+
if isinstance(obj, list):
198+
return obj
199+
elif obj is not None:
200+
return [obj]
201+
else:
202+
return []
203+
200204

205+
def get_node_constructor_wrapper(decorated_function): # noqa C901
201206
try:
202207
sig = signature(decorated_function)
203208
try:

‎tests/mapping/__init__.py‎

Whitespace-only changes.

tests/transformation/test_parse_tree_to_ast_transformers.py renamed to tests/mapping/test_parse_tree_to_ast_transformers.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,8 @@ class ParseTreeToASTTransformerTest(unittest.TestCase):
4949

5050
def test_simple_entities_transformer(self):
5151
transformer = ParseTreeToASTTransformer(allow_generic_node=False)
52-
transformer.register_node_factory(AntlrEntityParser.ModuleContext, lambda ctx: EModule(name=ctx.name.text))\
53-
.with_child(AntlrEntityParser.ModuleContext.entity, PropertyRef("entities"))
52+
transformer.register_node_factory(AntlrEntityParser.ModuleContext, lambda ctx: EModule(name=ctx.name.text)) \
53+
.with_child(PropertyRef("entities"), AntlrEntityParser.ModuleContext.entity)
5454
transformer.register_node_factory(AntlrEntityParser.EntityContext, lambda ctx: EEntity(name=ctx.name.text))
5555
expected_ast = EModule("M", [EEntity("FOO", []), EEntity("BAR", [])])
5656
actual_ast = transformer.transform(self.parse_entities("""
@@ -64,11 +64,11 @@ def test_simple_entities_transformer(self):
6464
def test_entities_with_features_transformer(self):
6565
transformer = ParseTreeToASTTransformer(allow_generic_node=False)
6666
transformer.register_node_factory(AntlrEntityParser.ModuleContext, lambda ctx: EModule(name=ctx.name.text)) \
67-
.with_child(AntlrEntityParser.ModuleContext.entity, PropertyRef("entities"))
67+
.with_child(PropertyRef("entities"), AntlrEntityParser.ModuleContext.entity)
6868
transformer.register_node_factory(AntlrEntityParser.EntityContext, lambda ctx: EEntity(name=ctx.name.text)) \
69-
.with_child(AntlrEntityParser.EntityContext.feature, PropertyRef("features"))
70-
transformer.register_node_factory(AntlrEntityParser.FeatureContext, lambda ctx: EFeature(name=ctx.name.text))\
71-
.with_child(AntlrEntityParser.FeatureContext.type_spec, PropertyRef("type"))
69+
.with_child(PropertyRef("features"), AntlrEntityParser.EntityContext.feature)
70+
transformer.register_node_factory(AntlrEntityParser.FeatureContext, lambda ctx: EFeature(name=ctx.name.text)) \
71+
.with_child(PropertyRef("type"), AntlrEntityParser.FeatureContext.type_spec)
7272
transformer.register_node_factory(AntlrEntityParser.Boolean_typeContext, EBooleanType)
7373
transformer.register_node_factory(AntlrEntityParser.String_typeContext, EStringType)
7474
transformer.register_node_factory(

‎tests/test_model.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from typing import List
44

55
from pylasu.model import Node, Position, Point
6-
from pylasu.model.model import Multiplicity
6+
from pylasu.model.reflection import Multiplicity
77
from pylasu.model.naming import ReferenceByName, Named, Scope, Symbol
88

99

‎tests/transformation/__init__.py‎

Whitespace-only changes.

‎tests/transformation/test_ast_transformers.py‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
from typing import List
55

66
from pylasu.model import Node
7+
from pylasu.testing.testing import assert_asts_are_equal
8+
from pylasu.transformation.generic_nodes import GenericNode
79
from pylasu.transformation.transformation import ASTTransformer, PropertyRef, ast_transformer
810

911

@@ -112,6 +114,45 @@ def test_translate_across_languages(self):
112114
),
113115
ALangIntLiteral(4))))
114116

117+
def test_dropping_nodes(self):
118+
prop = PropertyRef("statements")
119+
transformer = ASTTransformer()
120+
transformer.register_node_factory(CU, CU).with_child(prop, prop)
121+
transformer.register_node_factory(DisplayIntStatement, lambda _: None)
122+
transformer.register_identity_transformation(SetStatement)
123+
cu = CU(statements=[DisplayIntStatement(value=456), SetStatement(variable="foo", value=123)])
124+
transformed_cu = transformer.transform(cu)
125+
# TODO not yet supported self.assertTrue(transformed_cu.hasValidParents())
126+
self.assertEqual(transformed_cu.origin, cu)
127+
self.assertEqual(1, len(transformed_cu.statements))
128+
assert_asts_are_equal(self, cu.statements[1], transformed_cu.statements[0])
129+
130+
def test_nested_origin(self):
131+
prop = PropertyRef("statements")
132+
transformer = ASTTransformer()
133+
transformer.register_node_factory(CU, CU).with_child(prop, prop)
134+
transformer.register_node_factory(DisplayIntStatement, lambda s: s.with_origin(GenericNode()))
135+
cu = CU(statements=[DisplayIntStatement(value=456)])
136+
transformed_cu = transformer.transform(cu)
137+
# TODO not yet supported self.assertTrue(transformed_cu.hasValidParents())
138+
self.assertEqual(transformed_cu.origin, cu)
139+
self.assertIsInstance(transformed_cu.statements[0].origin, GenericNode)
140+
141+
def test_transforming_one_node_to_many(self):
142+
prop = PropertyRef("stmts")
143+
transformer = ASTTransformer(allow_generic_node=False)
144+
transformer.register_node_factory(BarRoot, BazRoot).with_child(prop, prop)
145+
transformer.register_node_factory(BarStmt, lambda s: [BazStmt(f"{s.desc}-1"), BazStmt(f"{s.desc}-2")])
146+
original = BarRoot([BarStmt("a"), BarStmt("b")])
147+
transformed = transformer.transform(original)
148+
# TODO not yet supported assertTrue { transformed.hasValidParents() }
149+
self.assertEquals(transformed.origin, original)
150+
assert_asts_are_equal(
151+
self,
152+
BazRoot([BazStmt("a-1"), BazStmt("a-2"), BazStmt("b-1"), BazStmt("b-2")]),
153+
transformed
154+
)
155+
115156

116157
@dataclass
117158
class ALangExpression(Node):
@@ -157,5 +198,25 @@ class BLangMult(BLangExpression):
157198
right: BLangExpression
158199

159200

201+
@dataclass
202+
class BarStmt(Node):
203+
desc: str
204+
205+
206+
@dataclass
207+
class BarRoot(Node):
208+
stmts: List[BarStmt] = field(default_factory=list)
209+
210+
211+
@dataclass
212+
class BazStmt(Node):
213+
desc: str
214+
215+
216+
@dataclass
217+
class BazRoot(Node):
218+
stmts: List[BazStmt] = field(default_factory=list)
219+
220+
160221
if __name__ == '__main__':
161222
unittest.main()

0 commit comments

Comments
 (0)