Skip to content

Add unit tests for parsing function and update doctest #331

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
PurpleGuitar merged 1 commit into from
Aug 20, 2026
Merged

Add unit tests for parsing function and update doctest #331

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 52 additions & 15 deletions backend/doc/domain/parsing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1453,26 +1453,61 @@ def split_chapter_into_verses_with_formatting(

>>> html_content = '''
... <span class="verse">
... <sup class="versemarker">19</sup>
... For through the law I died to the law, so that I might live for God. I have been crucified with Christ.
... <sup id="footnote-caller-1" class="caller"><a href="#footnote-target-1">1</a></sup>
... <div class="sectionhead-5"></div>
... <sup class="versemarker">1</sup>
... <span class="word-entry"> Généalogie </span>
... <span class="word-entry"> </span>
... de
... <span class="word-entry"> Jésus </span>
... -
... <span class="word-entry"> Christ </span>
... ,
... <span class="word-entry"> fils </span>
... de
... <span class="word-entry"> David </span>
... ,
... <span class="word-entry"> fils </span>
... d'
... <span class="word-entry"> Abraham </span>
... .
...
... </span>
... <span class="verse">
... <sup class="versemarker">20</sup>
... I have been crucified with Christ and I no longer live, but Christ lives in me. The life I now live in the body, I live by faith in the Son of God, who loved me and gave himself for me.
... <sup id="footnote-caller-2" class="caller"><a href="#footnote-target-2">2</a></sup>
... <div class="sectionhead-5"></div>
... <sup class="versemarker">2</sup>
... <span class="word-entry"> Abraham </span>
...
... <span class="word-entry"> engendra </span>
...
... <span class="word-entry"> Isaac </span>
... ;
... <span class="word-entry"> </span>
...
... <span class="word-entry"> Isaac </span>
...
... <span class="word-entry"> engendra </span>
...
... <span class="word-entry"> Jacob </span>
... ;
... <span class="word-entry"> </span>
...
... <span class="word-entry"> Jacob </span>
...
... <span class="word-entry"> engendra </span>
...
... <span class="word-entry"> Juda </span>
...
... <span class="word-entry"> et </span>
...
... <span class="word-entry"> ses </span>
...
... <span class="word-entry"> frères </span>
... ;
... </span>
... '''
>>> from doc.domain.parsing import split_chapter_into_verses_with_formatting
>>> chapter = USFMChapter(content=html_content, verses=None)
>>> chapter.verses = split_chapter_into_verses_with_formatting(chapter)
>>> print(chapter.verses["19"])
<sup class="versemarker">19</sup>
For through the law I died to the law, so that I might live for God. I have been crucified with Christ.
<sup id="footnote-caller-1" class="caller"><a href="#footnote-target-1">1</a></sup>
<BLANKLINE>
>>> print(chapter.verses["1"])
<span class="verse"> Généalogie de Jésus-Christ, fils de David, fils d'Abraham. </span>
"""
soup = BeautifulSoup(chapter.content, "html.parser")
verse_dict: dict[VerseRef, str] = {}
Expand Down Expand Up @@ -1505,9 +1540,11 @@ def clean_content_html(raw_content: str) -> str:
if __name__ == "__main__":

# To run the doctests in this module, in the root of the project do:
# python backend/document/domain/resource_lookup.py
# PYTHONPATH=backend python backend/doc/domain/parsing.py
# or
# python backend/document/domain/resource_lookup.py -v
# PYTHONPATH=backend python backend/doc/domain/parsing.py -v
# These doctests are not collected by pytest: pyproject.toml sets
# testpaths = ["tests"] with no doctest collection, so run them by hand.
# See https://docs.python.org/3/library/doctest.html
# for more details.
import doctest
Expand Down
140 changes: 140 additions & 0 deletions tests/unit/test_parsing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@
ensure_chapter_label,
ensure_chapter_marker,
maybe_localized_book_name,
split_chapter_into_verses_with_formatting,
)
from doc.domain import model, resource_lookup
from doc.domain.model import USFMChapter


def test_ensure_chapter_marker_unchanged_if_exists() -> None:
Expand Down Expand Up @@ -102,5 +104,143 @@ def test_fr_f10_book_name_lookup_prefs() -> None:
assert localized_book_name == expected


# Sample content taken from fr f10 Matthew 1, which uses word-entry tags.
FRENCH_WORD_ENTRY_HTML = """
<span class="verse">
<sup class="versemarker">1</sup>
<span class="word-entry"> Généalogie </span>
<span class="word-entry"> </span>
de
<span class="word-entry"> Jésus </span>
-
<span class="word-entry"> Christ </span>
,
<span class="word-entry"> fils </span>
de
<span class="word-entry"> David </span>
,
<span class="word-entry"> fils </span>
d'
<span class="word-entry"> Abraham </span>
.

</span>
<span class="verse">
<sup class="versemarker">2</sup>
<span class="word-entry"> Abraham </span>

<span class="word-entry"> engendra </span>

<span class="word-entry"> Isaac </span>
;
</span>
"""

# Sample content in the shape produced for USFM without word-entry tags: a
# footnote caller sup and a trailing sectionhead div follow the verse text.
GALATIANS_HTML = """
<span class="verse">
<sup class="versemarker"> 19 </sup>
For through the law I died to the law, so that I might live for God.
<sup id="footnote-caller-1" class="caller"><a href="#footnote-target-1">1</a></sup>
<div class="sectionhead-5"></div>
</span>
<span class="verse">
<sup class="versemarker">20</sup>
I have been crucified with Christ and I no longer live.
<sup id="footnote-caller-2" class="caller"><a href="#footnote-target-2">2</a></sup>
<div class="sectionhead-5"></div>
</span>
"""


def test_split_chapter_into_verses_with_formatting_keys() -> None:
chapter = USFMChapter(content=GALATIANS_HTML, verses=None)
verses = split_chapter_into_verses_with_formatting(chapter)
assert list(verses.keys()) == ["19", "20"]


def test_split_chapter_into_verses_with_formatting_strips_verse_number_whitespace() -> (
None
):
# The versemarker sup for verse 19 is written as "<sup ...> 19 </sup>".
chapter = USFMChapter(content=GALATIANS_HTML, verses=None)
verses = split_chapter_into_verses_with_formatting(chapter)
assert "19" in verses
assert " 19 " not in verses


def test_split_chapter_into_verses_with_formatting_removes_versemarker_only() -> None:
chapter = USFMChapter(content=GALATIANS_HTML, verses=None)
verses = split_chapter_into_verses_with_formatting(chapter)
for verse in verses.values():
assert "versemarker" not in verse
# Other markup inside the verse span survives.
assert verses["19"] == (
'<span class="verse"> For through the law I died to the law, '
"so that I might live for God.\n"
'<sup class="caller" id="footnote-caller-1">'
'<a href="#footnote-target-1">1</a></sup>\n'
'<div class="sectionhead-5"></div>\n</span>'
)


def test_split_chapter_into_verses_with_formatting_unwraps_word_entries() -> None:
chapter = USFMChapter(content=FRENCH_WORD_ENTRY_HTML, verses=None)
verses = split_chapter_into_verses_with_formatting(chapter)
assert list(verses.keys()) == ["1", "2"]
for verse in verses.values():
assert "word-entry" not in verse
# The wrapped text survives in place, with whitespace collapsed and
# whitespace before punctuation removed.
assert verses["1"] == (
'<span class="verse"> Généalogie de Jésus-Christ, fils de David, '
"fils d'Abraham. </span>"
)
assert verses["2"] == '<span class="verse"> Abraham engendra Isaac;\n</span>'


def test_split_chapter_into_verses_with_formatting_collapses_hyphen_spacing() -> None:
"""Spacing around a hyphen is intentionally collapsed (see clean_content_html)."""
chapter = USFMChapter(content=FRENCH_WORD_ENTRY_HTML, verses=None)
verses = split_chapter_into_verses_with_formatting(chapter)
assert "Jésus-Christ" in verses["1"]
assert "Jésus - Christ" not in verses["1"]


def test_split_chapter_into_verses_with_formatting_skips_verses_without_versemarker() -> (
None
):
html_content = """
<span class="verse">
No versemarker sup at all here.
</span>
<span class="verse">
<sup class="versemarker"></sup>
An empty versemarker sup here.
</span>
<span class="verse">
<sup class="versemarker">3</sup>
A well formed verse.
</span>
"""
chapter = USFMChapter(content=html_content, verses=None)
verses = split_chapter_into_verses_with_formatting(chapter)
assert list(verses.keys()) == ["3"]
assert verses["3"] == '<span class="verse"> A well formed verse.\n</span>'


def test_split_chapter_into_verses_with_formatting_without_verse_spans() -> None:
chapter = USFMChapter(
content="<p>Chapter content with no verse spans.</p>", verses=None
)
assert split_chapter_into_verses_with_formatting(chapter) == {}


def test_split_chapter_into_verses_with_formatting_empty_content() -> None:
chapter = USFMChapter(content="", verses=None)
assert split_chapter_into_verses_with_formatting(chapter) == {}


if __name__ == "__main__":
pytest.main()
Loading