Skip to content

Commit 0dc1ec4

Browse files
committed
refactor: use plain-parser for spec parsing
Parser code moved to the plain-parser package. plain_file, plain_spec, concept_utils and plain2code_nodes are now re-exports. Shim removal is a follow-up. The missing linked resource error now also reports the module name. Includes the requires-without-local-definitions fix from f5fd1c5, now in plain-parser.
1 parent 8a35fd2 commit 0dc1ec4

108 files changed

Lines changed: 152 additions & 4130 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎concept_utils.py‎

Lines changed: 2 additions & 220 deletions
Original file line numberDiff line numberDiff line change
@@ -1,221 +1,3 @@
1-
import re
2-
from copy import deepcopy
3-
from typing import Optional
1+
"""Compatibility shim: the parser now lives in the ``plain-parser`` package."""
42

5-
import mistletoe.block_token
6-
import networkx as nx
7-
8-
import plain_spec
9-
from plain2code_exceptions import PlainSyntaxError
10-
11-
# These are reserved concept names that are pre-defined and must not be redefined in the definitions section.
12-
DEFAULT_CONCEPTS = {
13-
":ConformanceTests:",
14-
":UnitTests:",
15-
":AcceptanceTest:",
16-
":AcceptanceTests:",
17-
":Implementation:",
18-
":plainDefinitions:",
19-
":plainImplementationReqs:",
20-
":plainFunctionality:",
21-
":plainTestReqs:",
22-
":plainImplementationCode:",
23-
}
24-
25-
26-
def extract_concepts_from_definition(text: str) -> tuple[list[str], list[str]]:
27-
pattern = r"-\s(:[^\:]+:)(?:,\s*:[^\:]+:)*"
28-
match = re.search(pattern, text)
29-
30-
if not match:
31-
return list(), [
32-
f"Syntax error: Invalid definition specification text: {text}. Should start with `- :<concept>:` (where <concept> is any string that does not contain `:`)."
33-
]
34-
35-
concept_candidates = re.findall(r":[^\:]+:", match.group())
36-
valid_concepts = list[str]()
37-
concept_re = r":[+\-\.0-9A-Z_a-z]+:"
38-
errors = list[str]()
39-
for candidate in concept_candidates:
40-
if re.match(concept_re, candidate):
41-
valid_concepts.append(candidate)
42-
else:
43-
errors.append(
44-
f"Syntax error: Invalid concept: {candidate}. Should contain only letters, numbers, hyphens and dots."
45-
)
46-
47-
return valid_concepts, errors
48-
49-
50-
def extract_concepts_from_spec_text(text: str) -> str:
51-
return re.findall(r":[+\-\.0-9A-Z_a-z]+:", text)
52-
53-
54-
def collect_concepts(plain_source) -> tuple[set[str], list[str]]:
55-
if plain_spec.DEFINITIONS not in plain_source:
56-
return set(), list()
57-
58-
concepts = list[str]()
59-
errors = list[str]()
60-
for definition in plain_source[plain_spec.DEFINITIONS]:
61-
spec_text = definition["markdown"]
62-
new_concepts, new_errors = extract_concepts_from_definition(spec_text)
63-
errors.extend(new_errors)
64-
concepts.extend(new_concepts)
65-
66-
return concepts, errors
67-
68-
69-
def validate_concepts_in_spec(spec: dict, concepts: set[str], spec_group: str) -> list[str]:
70-
spec_text = spec["markdown"]
71-
used_concepts = extract_concepts_from_spec_text(spec_text)
72-
errors = []
73-
for used_concept in used_concepts:
74-
if used_concept not in concepts and used_concept not in DEFAULT_CONCEPTS:
75-
errors.append(
76-
f"Syntax error: Concept {used_concept} is not defined in the definitions. {spec_group}: {spec_text}"
77-
)
78-
79-
return errors
80-
81-
82-
def validate_concepts_in_spec_group(spec_group, plain_source, concepts: set[str]) -> list[str]:
83-
if spec_group not in plain_source:
84-
return []
85-
86-
errors = []
87-
for spec in plain_source[spec_group]:
88-
errors.extend(validate_concepts_in_spec(spec, concepts, spec_group))
89-
90-
return errors
91-
92-
93-
def validate_concepts(marshalled_plain_source) -> list[str]:
94-
errors = list[str]()
95-
concepts = deepcopy(DEFAULT_CONCEPTS)
96-
tree_based_concepts = deepcopy(DEFAULT_CONCEPTS)
97-
98-
new_concepts, new_errors = collect_concepts(marshalled_plain_source)
99-
errors.extend(new_errors)
100-
101-
unique_new_concepts = set(new_concepts)
102-
intersection = unique_new_concepts.intersection(concepts)
103-
104-
if len(unique_new_concepts) < len(new_concepts) or (len(intersection) > 0):
105-
misused_concepts = {concept for concept in new_concepts if new_concepts.count(concept) > 1}
106-
misused_concepts.update(intersection)
107-
errors.append(f"Syntax error: Concepts were defined multiple times: {', '.join(misused_concepts)} .")
108-
109-
concepts.update(unique_new_concepts)
110-
tmp_tree_based_concepts = tree_based_concepts.union(unique_new_concepts)
111-
112-
for spec_group in [
113-
plain_spec.NON_FUNCTIONAL_REQUIREMENTS,
114-
plain_spec.TEST_REQUIREMENTS,
115-
plain_spec.FUNCTIONAL_REQUIREMENTS,
116-
plain_spec.DEFINITIONS,
117-
]:
118-
errors.extend(validate_concepts_in_spec_group(spec_group, marshalled_plain_source, tmp_tree_based_concepts))
119-
120-
if plain_spec.FUNCTIONAL_REQUIREMENTS in marshalled_plain_source:
121-
for func_spec in marshalled_plain_source[plain_spec.FUNCTIONAL_REQUIREMENTS]:
122-
if "acceptance_tests" not in func_spec:
123-
continue
124-
125-
acceptance_tests = func_spec["acceptance_tests"]
126-
for spec in acceptance_tests:
127-
errors.extend(validate_concepts_in_spec(spec, tmp_tree_based_concepts, plain_spec.ACCEPTANCE_TESTS))
128-
129-
return errors
130-
131-
132-
def find_concept_definitions_in_plain_source(
133-
concept: str,
134-
plain_source: dict,
135-
renderer,
136-
) -> list[mistletoe.block_token.token]:
137-
definitions = list[mistletoe.block_token.token]()
138-
if plain_source[plain_spec.DEFINITIONS] is None:
139-
return definitions
140-
141-
for definition in plain_source[plain_spec.DEFINITIONS].children:
142-
rendered_definition = renderer.render(definition)
143-
defined_concepts, _ = extract_concepts_from_definition(rendered_definition)
144-
if concept in defined_concepts:
145-
definitions.append(definition)
146-
used_concepts = [
147-
used_concept
148-
for used_concept in extract_concepts_from_spec_text(rendered_definition)
149-
if used_concept != concept and used_concept not in DEFAULT_CONCEPTS
150-
]
151-
for used_concept in used_concepts:
152-
definitions.extend(find_concept_definitions_in_plain_source(used_concept, plain_source, renderer))
153-
154-
return definitions
155-
156-
157-
def build_adjacency_list(definitions: Optional[list[dict]]) -> tuple[dict, dict]:
158-
adjacency_list = dict[str, list[str]]()
159-
concept_definitions = dict[str, dict]()
160-
if definitions is None or len(definitions) == 0:
161-
return adjacency_list, concept_definitions
162-
163-
for definition in definitions:
164-
def_text = definition["markdown"]
165-
new_concepts, _ = extract_concepts_from_definition(def_text)
166-
used_concepts = [
167-
used_concept
168-
for used_concept in extract_concepts_from_spec_text(def_text)
169-
if used_concept not in new_concepts and used_concept not in DEFAULT_CONCEPTS
170-
]
171-
used_concepts = set(used_concepts)
172-
173-
for new_concept in new_concepts:
174-
if new_concept not in adjacency_list:
175-
adjacency_list[new_concept] = []
176-
177-
for used_concept in used_concepts:
178-
if used_concept not in adjacency_list:
179-
adjacency_list[used_concept] = []
180-
181-
adjacency_list[used_concept].extend(new_concepts)
182-
183-
for new_concept in new_concepts:
184-
concept_definitions[new_concept] = definition
185-
186-
return adjacency_list, concept_definitions
187-
188-
189-
def sort_definitions(definitions: list[dict]) -> list[dict]:
190-
if len(definitions) <= 1:
191-
return
192-
193-
adjacency_list, concept_definitions = build_adjacency_list(definitions)
194-
195-
concept_graph = nx.DiGraph()
196-
for node, neighbors in adjacency_list.items():
197-
for neighbour in neighbors:
198-
concept_graph.add_edge(node, neighbour)
199-
if not neighbors:
200-
concept_graph.add_node(node)
201-
202-
if not nx.is_directed_acyclic_graph(concept_graph):
203-
msg = "Found cycles in the concept graph. Cycles are not allowed."
204-
all_cycles = list(nx.simple_cycles(concept_graph))
205-
206-
for cycle in all_cycles:
207-
cyclic_definitions = []
208-
cyclic_definitions.append(concept_definitions[cycle[0]]["markdown"])
209-
cyclic_definitions.append(concept_definitions[cycle[1]]["markdown"])
210-
msg += "Cyclic definitons:\n"
211-
msg += "\n".join(cyclic_definitions)
212-
msg += "\n"
213-
214-
raise PlainSyntaxError(f"Plain syntax error: {msg}")
215-
216-
order = list(nx.topological_sort(concept_graph))
217-
if len(order) > 0:
218-
definition_to_order_idx = {
219-
plain_spec.hash_text(str(concept_definitions[concept])): idx for idx, concept in enumerate(order)
220-
}
221-
definitions.sort(key=lambda definition: definition_to_order_idx[plain_spec.hash_text(str(definition))])
3+
from plain_parser.concept_utils import * # noqa: F401,F403

‎file_utils.py‎

Lines changed: 6 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,14 @@
33
import stat
44
from pathlib import Path
55

6-
from liquid2 import Environment, FileSystemLoader, StrictUndefined
7-
from liquid2.exceptions import UndefinedError
6+
from plain_parser.loaders import ( # noqa: F401
7+
TrackingFileSystemLoader,
8+
get_loaded_templates,
9+
load_linked_resources,
10+
open_from,
11+
)
812

9-
import plain_spec
1013
from plain2code_console import console
11-
from plain2code_exceptions import UnsupportedBase64Content, UnsupportedResourceType
12-
from plain2code_nodes import Plain2CodeIncludeTag, Plain2CodeLoaderMixin
13-
from plain2code_utils import find_large_base64_blob
1414
from plain_modules import CODEPLAIN_MEMORY_SUBFOLDER, CODEPLAIN_METADATA_FOLDER
1515

1616
BINARY_FILE_EXTENSIONS = [".pyc"]
@@ -167,106 +167,6 @@ def store_response_files(target_folder, response_files, existing_files):
167167
return existing_files
168168

169169

170-
def open_from(dirs, file_name):
171-
for dir in dirs:
172-
full_file_name = os.path.join(dir, file_name)
173-
if not os.path.isfile(full_file_name):
174-
continue
175-
176-
with open(full_file_name, "rb") as f:
177-
content = f.read()
178-
return content.decode("utf-8")
179-
180-
return None
181-
182-
183-
def resolve_linked_resource(dirs, file_name):
184-
"""Return the first existing path for file_name across dirs, or None if not found.
185-
186-
Mirrors the search order used by open_from so that validating a linked resource and
187-
loading it can never disagree.
188-
"""
189-
for dir in dirs:
190-
full_file_name = os.path.join(dir, file_name)
191-
if os.path.exists(full_file_name):
192-
return full_file_name
193-
194-
return None
195-
196-
197-
def load_linked_resources(template_dirs: list[str], resources_list, module_name: str):
198-
linked_resources = {}
199-
200-
for resource in resources_list:
201-
file_name = resource["target"]
202-
if file_name in linked_resources:
203-
continue
204-
205-
try:
206-
content = open_from(template_dirs, file_name)
207-
except UnicodeDecodeError:
208-
raise UnsupportedResourceType(
209-
f"Referenced resource '{file_name}' in module '{module_name}' is a binary file. "
210-
f"Only text files (e.g. .md, .txt, .json, .yaml) can be referenced from a .plain file."
211-
)
212-
213-
if content is None:
214-
raise FileNotFoundError(f"""
215-
Resource file {file_name} not found. Resource files are searched in the following order (highest to lowest precedence):
216-
217-
1. The directory containing your .plain file
218-
2. The directory specified by --template-dir (if provided)
219-
3. The built-in 'standard_template_library' directory
220-
221-
Please ensure that the resource exists in one of these locations, or specify the correct --template-dir if using custom templates.
222-
""")
223-
224-
blob = find_large_base64_blob(content)
225-
if blob is not None:
226-
raise UnsupportedBase64Content(
227-
f"Referenced resource '{file_name}' in module '{module_name}' contains a large "
228-
f"base64-encoded blob ({len(blob)} characters), such as an embedded image. Inline "
229-
"base64 data is not supported. Remove the data from the resource. "
230-
"If the data should be used by the end software, "
231-
"save the data to a separate file and include the file path in the specification without it being a reference file."
232-
)
233-
234-
linked_resources[file_name] = content
235-
236-
return linked_resources
237-
238-
239-
class TrackingFileSystemLoader(Plain2CodeLoaderMixin, FileSystemLoader):
240-
def __init__(self, *args, **kwargs):
241-
super().__init__(*args, **kwargs)
242-
self.loaded_templates = {}
243-
244-
def get_source(self, environment, template_name, **kwargs):
245-
source = super().get_source(environment, template_name, **kwargs)
246-
self.loaded_templates[template_name] = source.source
247-
return source
248-
249-
250-
def get_loaded_templates(source_path, plain_source):
251-
# Render the plain source with Liquid templating engine
252-
# to identify the templates that are being loaded
253-
254-
liquid_loader = TrackingFileSystemLoader(source_path)
255-
liquid_env = Environment(loader=liquid_loader, undefined=StrictUndefined)
256-
liquid_env.tags["include"] = Plain2CodeIncludeTag(liquid_env)
257-
258-
liquid_env.filters["code_variable"] = plain_spec.code_variable_liquid_filter
259-
liquid_env.filters["prohibited_chars"] = plain_spec.prohibited_chars_liquid_filter
260-
261-
plain_source_template = liquid_env.from_string(plain_source)
262-
try:
263-
plain_source = plain_source_template.render()
264-
except UndefinedError as e:
265-
raise Exception(f"Undefined liquid variable: {str(e)}")
266-
267-
return plain_source, liquid_loader.loaded_templates
268-
269-
270170
def update_build_folder_with_rendered_files(build_folder, existing_files, response_files):
271171
changed_files = set()
272172
changed_files.update(response_files.keys())

0 commit comments

Comments
 (0)