From e10540b111135e0ed639951cf472d3363257af89 Mon Sep 17 00:00:00 2001 From: Rossi-Luciano Date: Fri, 3 Jul 2026 20:04:11 -0300 Subject: [PATCH 1/5] =?UTF-8?q?feat(pubmed):=20envolver=20sa=C3=ADda=20em?= =?UTF-8?q?=20ArticleSet/DOCTYPE=20e=20corrigir=20ordem=20dos=20elementos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pipeline_pubmed() gerava um
solto, sem o envelope e DOCTYPE exigidos pela DTD do PubMed. Extrai a construção do
para build_pubmed_article() e adiciona pipeline_pubmed_set() para agrupar múltiplos artigos num único (pré-requisito da #1241). Validação contra a DTD oficial (lxml.etree.DTD + https://dtd.nlm.nih.gov/ncbi/pubmed/in/PubMed.dtd) expôs 2 bugs no pipeline, corrigidos aqui: Abstract/OtherAbstract precisavam vir antes de CopyrightInformation/CoiStatement/ObjectList/ReferenceList, e VernacularTitle nunca era chamado apesar de implementado e testado. Refs #1226, #1234 --- packtools/sps/formats/pubmed.py | 47 ++++++++++++++++++++++++++++---- tests/sps/formats/test_pubmed.py | 35 ++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/packtools/sps/formats/pubmed.py b/packtools/sps/formats/pubmed.py index f99b2651d..89329d84f 100644 --- a/packtools/sps/formats/pubmed.py +++ b/packtools/sps/formats/pubmed.py @@ -256,7 +256,20 @@ def xml_pubmed_language_pipe(xml_pubmed, xml_tree): add_langs(xml_pubmed, xml_tree) -def pipeline_pubmed(xml_tree, pretty_print=True): +PUBMED_DOCTYPE = ( + '' +) + + +def xml_pubmed_article_set_pipe(): + return ET.Element("ArticleSet") + + +def build_pubmed_article(xml_tree): + """ + Builds a single
element for one SciELO XML tree. + """ xml_pubmed = xml_pubmed_article_pipe() xml_pubmed_journal_pipe(xml_pubmed) xml_pubmed_publisher_name_pipe(xml_pubmed, xml_tree) @@ -266,6 +279,7 @@ def pipeline_pubmed(xml_tree, pretty_print=True): xml_pubmed_issue_pipe(xml_pubmed, xml_tree) xml_pubmed_pub_date_pipe(xml_pubmed, xml_tree) xml_pubmed_article_title_pipe(xml_pubmed, xml_tree) + xml_pubmed_vernacular_title_pipe(xml_pubmed, xml_tree) xml_pubmed_first_page_pipe(xml_pubmed, xml_tree) xml_pubmed_elocation_pipe(xml_pubmed, xml_tree) xml_pubmed_language_pipe(xml_pubmed, xml_tree) @@ -273,16 +287,39 @@ def pipeline_pubmed(xml_tree, pretty_print=True): xml_pubmed_publication_type(xml_pubmed, xml_tree) xml_pubmed_article_id(xml_pubmed, xml_tree) xml_pubmed_history(xml_pubmed, xml_tree) + xml_pubmed_abstract(xml_pubmed, xml_tree) + xml_pubmed_other_abstract(xml_pubmed, xml_tree) xml_pubmed_copyright_information(xml_pubmed, xml_tree) xml_pubmed_coi_statement(xml_pubmed, xml_tree) xml_pubmed_object_list(xml_pubmed, xml_tree) xml_pubmed_title_reference_list(xml_pubmed, xml_tree) xml_pubmed_citations(xml_pubmed, xml_tree) - xml_pubmed_abstract(xml_pubmed, xml_tree) - xml_pubmed_other_abstract(xml_pubmed, xml_tree) + return xml_pubmed + + +def pipeline_pubmed_set(xml_trees, pretty_print=True): + """ + Builds a complete PubMed XML document ( + + ) containing one
per SciELO XML tree in xml_trees. + """ + article_set = xml_pubmed_article_set_pipe() + for xml_tree in xml_trees: + article_set.append(build_pubmed_article(xml_tree)) - xml_tree = ET.ElementTree(xml_pubmed) - return ET.tostring(xml_tree, pretty_print=pretty_print, encoding="utf-8").decode("utf-8") + return ET.tostring( + ET.ElementTree(article_set), + pretty_print=pretty_print, + encoding="utf-8", + xml_declaration=True, + doctype=PUBMED_DOCTYPE, + ).decode("utf-8") + + +def pipeline_pubmed(xml_tree, pretty_print=True): + """ + Builds a complete PubMed XML document for a single SciELO XML tree. + """ + return pipeline_pubmed_set([xml_tree], pretty_print=pretty_print) def get_authors(xml_tree): diff --git a/tests/sps/formats/test_pubmed.py b/tests/sps/formats/test_pubmed.py index a8afad598..0788d733d 100644 --- a/tests/sps/formats/test_pubmed.py +++ b/tests/sps/formats/test_pubmed.py @@ -25,6 +25,9 @@ xml_pubmed_citations, xml_pubmed_abstract, xml_pubmed_other_abstract, + pipeline_pubmed, + pipeline_pubmed_set, + PUBMED_DOCTYPE, ) @@ -1827,6 +1830,38 @@ def test_xml_pubmed_other_abstract(self): self.assertEqual(obtained, expected) + def test_pipeline_pubmed_wraps_article_in_article_set_with_doctype(self): + xml_tree = ET.fromstring( + '
' + '' + '
' + ) + + obtained = pipeline_pubmed(xml_tree, pretty_print=False) + + self.assertTrue(obtained.startswith("\n")) + self.assertIn(PUBMED_DOCTYPE, obtained) + self.assertIn('
', obtained) + self.assertTrue(obtained.rstrip().endswith('
')) + self.assertEqual(obtained.count('
'), 1) + + def test_pipeline_pubmed_set_wraps_multiple_articles_in_single_article_set(self): + xml_tree = ET.fromstring( + '
' + '' + '
' + ) + + obtained = pipeline_pubmed_set([xml_tree, xml_tree], pretty_print=False) + + self.assertIn(PUBMED_DOCTYPE, obtained) + self.assertEqual(obtained.count(''), 1) + self.assertEqual(obtained.count('
'), 2) + self.assertEqual(obtained.count('
'), 2) if __name__ == '__main__': From 0b7810b2708307cc414fce97df3f996c165caa99 Mon Sep 17 00:00:00 2001 From: Rossi-Luciano Date: Mon, 6 Jul 2026 10:34:21 -0300 Subject: [PATCH 2/5] =?UTF-8?q?fix(pubmed):=20trata=20aus=C3=AAncia=20de?= =?UTF-8?q?=20PubDate=20como=20erro=20em=20build=5Fpubmed=5Farticle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reconcilia #1242 com o envelope ArticleSet desta branch. Journal/PubDate é obrigatório na DTD do PubMed; build_pubmed_article agora levanta MissingRequiredPubDateError quando não encontra data, em vez de gerar um Journal inválido. pipeline_pubmed_set continua estrito (propaga o erro para qualquer artigo inválido do lote). Extrai build_article_set_xml (wrap de
s já construídos em + DOCTYPE) de dentro de pipeline_pubmed_set, para uso por chamadores que precisem filtrar artigos inválidos antes de montar o documento final (caso do CLI em lote da #1241). Co-Authored-By: Claude Sonnet 5 --- packtools/sps/formats/pubmed.py | 37 ++++++++++++++--- tests/sps/formats/test_pubmed.py | 69 +++++++++++++++++++++++++++++++- 2 files changed, 99 insertions(+), 7 deletions(-) diff --git a/packtools/sps/formats/pubmed.py b/packtools/sps/formats/pubmed.py index 89329d84f..8d2c5898d 100644 --- a/packtools/sps/formats/pubmed.py +++ b/packtools/sps/formats/pubmed.py @@ -16,6 +16,12 @@ ) +class MissingRequiredPubDateError(Exception): + """Raised when a SciELO article has no usable publication date, so the + DTD-required PubMed.dtd `Journal/PubDate` element (`(..., PubDate)`, + no `?`) cannot be filled in.""" + + def xml_pubmed_article_pipe(): return ET.Element("Article") @@ -278,6 +284,11 @@ def build_pubmed_article(xml_tree): xml_pubmed_volume_pipe(xml_pubmed, xml_tree) xml_pubmed_issue_pipe(xml_pubmed, xml_tree) xml_pubmed_pub_date_pipe(xml_pubmed, xml_tree) + if xml_pubmed.find("Journal/PubDate") is None: + raise MissingRequiredPubDateError( + "Journal/PubDate é obrigatório na DTD do PubMed e não foi " + "possível determinar uma data de publicação para este artigo." + ) xml_pubmed_article_title_pipe(xml_pubmed, xml_tree) xml_pubmed_vernacular_title_pipe(xml_pubmed, xml_tree) xml_pubmed_first_page_pipe(xml_pubmed, xml_tree) @@ -297,14 +308,15 @@ def build_pubmed_article(xml_tree): return xml_pubmed -def pipeline_pubmed_set(xml_trees, pretty_print=True): +def build_article_set_xml(articles, pretty_print=True): """ - Builds a complete PubMed XML document ( + - ) containing one
per SciELO XML tree in xml_trees. + Wraps a list of already-built
elements (see build_pubmed_article) + into the complete PubMed XML document ( + + ). """ article_set = xml_pubmed_article_set_pipe() - for xml_tree in xml_trees: - article_set.append(build_pubmed_article(xml_tree)) + for article in articles: + article_set.append(article) return ET.tostring( ET.ElementTree(article_set), @@ -315,6 +327,21 @@ def pipeline_pubmed_set(xml_trees, pretty_print=True): ).decode("utf-8") +def pipeline_pubmed_set(xml_trees, pretty_print=True): + """ + Builds a complete PubMed XML document ( + + ) containing one
per SciELO XML tree in xml_trees. + + Raises MissingRequiredPubDateError immediately if any xml_tree has no + usable publication date. Callers that want to skip such articles + instead of failing the whole batch (e.g. the batch CLI) should build + each
themselves via build_pubmed_article and assemble the + final document with build_article_set_xml. + """ + articles = [build_pubmed_article(xml_tree) for xml_tree in xml_trees] + return build_article_set_xml(articles, pretty_print=pretty_print) + + def pipeline_pubmed(xml_tree, pretty_print=True): """ Builds a complete PubMed XML document for a single SciELO XML tree. diff --git a/tests/sps/formats/test_pubmed.py b/tests/sps/formats/test_pubmed.py index 0788d733d..c981c7f9f 100644 --- a/tests/sps/formats/test_pubmed.py +++ b/tests/sps/formats/test_pubmed.py @@ -27,7 +27,10 @@ xml_pubmed_other_abstract, pipeline_pubmed, pipeline_pubmed_set, + build_pubmed_article, + build_article_set_xml, PUBMED_DOCTYPE, + MissingRequiredPubDateError, ) @@ -1835,7 +1838,11 @@ def test_pipeline_pubmed_wraps_article_in_article_set_with_doctype(self): '
' - '' + '' + '' + '06012023' + '' + '' '
' ) @@ -1852,7 +1859,11 @@ def test_pipeline_pubmed_set_wraps_multiple_articles_in_single_article_set(self) '
' - '' + '' + '' + '06012023' + '' + '' '
' ) @@ -1863,6 +1874,60 @@ def test_pipeline_pubmed_set_wraps_multiple_articles_in_single_article_set(self) self.assertEqual(obtained.count('
'), 2) self.assertEqual(obtained.count('
'), 2) + def test_pipeline_pubmed_set_raises_when_any_article_has_no_pub_date(self): + xml_tree_with_date = ET.fromstring( + '
' + '' + '' + '06012023' + '' + '' + '
' + ) + xml_tree_without_date = ET.fromstring( + '
' + '' + '
' + ) + + with self.assertRaises(MissingRequiredPubDateError): + pipeline_pubmed_set([xml_tree_with_date, xml_tree_without_date]) + + def test_build_pubmed_article_raises_when_no_pub_date_is_found(self): + xml_tree = ET.fromstring( + '
' + '' + '
' + ) + + with self.assertRaises(MissingRequiredPubDateError): + build_pubmed_article(xml_tree) + + def test_build_article_set_xml_wraps_pre_built_articles(self): + xml_tree = ET.fromstring( + '
' + '' + '' + '06012023' + '' + '' + '
' + ) + article = build_pubmed_article(xml_tree) + + obtained = build_article_set_xml([article], pretty_print=False) + + self.assertIn(PUBMED_DOCTYPE, obtained) + self.assertEqual(obtained.count('
'), 1) + if __name__ == '__main__': unittest.main() From 2f59068f4e31b7652dd2f442d3097e243bb0cc85 Mon Sep 17 00:00:00 2001 From: Rossi-Luciano Date: Fri, 10 Jul 2026 13:23:14 -0300 Subject: [PATCH 3/5] refactor(pubmed): move required-PubDate check into a generic pipe-level error Per review on #1250: the required-element check belongs to the pipe function that builds the element, not bolted on after the fact in build_pubmed_article -- and a missing element reflects incomplete source data, not a transform bug. MissingRequiredPubDateError becomes MissingRequiredElementError(element_path), raised directly inside xml_pubmed_pub_date_pipe, ready to be reused for the other DTD-required Journal children (PublisherName, JournalTitle, Issn) in a follow-up. --- packtools/sps/formats/pubmed.py | 54 +++++++++++++++++--------------- tests/sps/formats/test_pubmed.py | 18 +++-------- 2 files changed, 34 insertions(+), 38 deletions(-) diff --git a/packtools/sps/formats/pubmed.py b/packtools/sps/formats/pubmed.py index 8d2c5898d..73f94ea27 100644 --- a/packtools/sps/formats/pubmed.py +++ b/packtools/sps/formats/pubmed.py @@ -16,10 +16,17 @@ ) -class MissingRequiredPubDateError(Exception): - """Raised when a SciELO article has no usable publication date, so the - DTD-required PubMed.dtd `Journal/PubDate` element (`(..., PubDate)`, - no `?`) cannot be filled in.""" +class MissingRequiredElementError(Exception): + """Raised when a SciELO article has no data for an element the PubMed + DTD marks as required (no `?`), so a valid
cannot be built + for it. `element_path` identifies which element (e.g. "Journal/PubDate").""" + + def __init__(self, element_path): + self.element_path = element_path + super().__init__( + f"{element_path} é obrigatório na DTD do PubMed e não foi " + "possível determinar seu valor para este artigo." + ) def xml_pubmed_article_pipe(): @@ -133,20 +140,21 @@ def xml_pubmed_pub_date_pipe(xml_pubmed, xml_tree): """ date = get_date(xml_tree) - if date is not None: - dt = ET.Element("PubDate") - dt.set("PubStatus", "epublish") - for element in ["year", "month", "day"]: - # TODO - # Season - # The season of publication. e.g.,Winter, Spring, Summer, Fall. Do not use if a Month is available. - # There is no example of using this value in the files. - - if date.get(element): - el = ET.Element(element.capitalize()) - el.text = date.get(element) - dt.append(el) - xml_pubmed.find("Journal").append(dt) + if date is None: + raise MissingRequiredElementError("Journal/PubDate") + dt = ET.Element("PubDate") + dt.set("PubStatus", "epublish") + for element in ["year", "month", "day"]: + # TODO + # Season + # The season of publication. e.g.,Winter, Spring, Summer, Fall. Do not use if a Month is available. + # There is no example of using this value in the files. + + if date.get(element): + el = ET.Element(element.capitalize()) + el.text = date.get(element) + dt.append(el) + xml_pubmed.find("Journal").append(dt) def xml_pubmed_replaces_pipe(xml_pubmed, xml_tree): @@ -284,11 +292,6 @@ def build_pubmed_article(xml_tree): xml_pubmed_volume_pipe(xml_pubmed, xml_tree) xml_pubmed_issue_pipe(xml_pubmed, xml_tree) xml_pubmed_pub_date_pipe(xml_pubmed, xml_tree) - if xml_pubmed.find("Journal/PubDate") is None: - raise MissingRequiredPubDateError( - "Journal/PubDate é obrigatório na DTD do PubMed e não foi " - "possível determinar uma data de publicação para este artigo." - ) xml_pubmed_article_title_pipe(xml_pubmed, xml_tree) xml_pubmed_vernacular_title_pipe(xml_pubmed, xml_tree) xml_pubmed_first_page_pipe(xml_pubmed, xml_tree) @@ -332,8 +335,9 @@ def pipeline_pubmed_set(xml_trees, pretty_print=True): Builds a complete PubMed XML document ( + ) containing one
per SciELO XML tree in xml_trees. - Raises MissingRequiredPubDateError immediately if any xml_tree has no - usable publication date. Callers that want to skip such articles + Raises MissingRequiredElementError immediately if any xml_tree is + missing data for a DTD-required element (e.g. no usable publication + date). Callers that want to skip such articles instead of failing the whole batch (e.g. the batch CLI) should build each
themselves via build_pubmed_article and assemble the final document with build_article_set_xml. diff --git a/tests/sps/formats/test_pubmed.py b/tests/sps/formats/test_pubmed.py index c981c7f9f..bd22f51b8 100644 --- a/tests/sps/formats/test_pubmed.py +++ b/tests/sps/formats/test_pubmed.py @@ -30,7 +30,7 @@ build_pubmed_article, build_article_set_xml, PUBMED_DOCTYPE, - MissingRequiredPubDateError, + MissingRequiredElementError, ) @@ -512,11 +512,6 @@ def test_xml_pubmed_pub_date_pipe_without_year(self): self.assertEqual(obtained, expected) def test_xml_pubmed_pub_date_pipe_without_date(self): - expected = ( - '
' - '' - '
' - ) xml_pubmed = ET.fromstring( '
' '' @@ -533,11 +528,8 @@ def test_xml_pubmed_pub_date_pipe_without_date(self): '
' ) - xml_pubmed_pub_date_pipe(xml_pubmed, xml_tree) - - obtained = ET.tostring(xml_pubmed, encoding="utf-8").decode("utf-8") - - self.assertEqual(obtained, expected) + with self.assertRaises(MissingRequiredElementError): + xml_pubmed_pub_date_pipe(xml_pubmed, xml_tree) def test_xml_pubmed_article_title_pipe(self): expected = ( @@ -1894,7 +1886,7 @@ def test_pipeline_pubmed_set_raises_when_any_article_has_no_pub_date(self): '
' ) - with self.assertRaises(MissingRequiredPubDateError): + with self.assertRaises(MissingRequiredElementError): pipeline_pubmed_set([xml_tree_with_date, xml_tree_without_date]) def test_build_pubmed_article_raises_when_no_pub_date_is_found(self): @@ -1906,7 +1898,7 @@ def test_build_pubmed_article_raises_when_no_pub_date_is_found(self): '
' ) - with self.assertRaises(MissingRequiredPubDateError): + with self.assertRaises(MissingRequiredElementError): build_pubmed_article(xml_tree) def test_build_article_set_xml_wraps_pre_built_articles(self): From 0d4b37c37377b3fbd1d87e9ee9a4ea1dce2b9800 Mon Sep 17 00:00:00 2001 From: Rossi-Luciano Date: Fri, 10 Jul 2026 15:14:51 -0300 Subject: [PATCH 4/5] =?UTF-8?q?fix(pubmed):=20corrige=20PubDate=20ausente,?= =?UTF-8?q?=20crash=20em=20citations=20e=20pagina=C3=A7=C3=A3o=20perdida?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Encontrados ao validar as 26 amostras reais de tests/samples/ contra a DTD real do PubMed em vez de só 1-2 arquivos escolhidos a dedo. 1. get_date() só olhava article_date (epub), ignorando collection_date (pub-type="epub-ppub"/date-type="collection") -- o padrão mais comum em artigos SciELO mais antigos. Isso descartava 21 de 26 (81%) das amostras reais do lote inteiro por "falta de PubDate", quando na verdade tinham data. references.py e oai_dc.py já fazem esse fallback manualmente; só pubmed.py não fazia. 2. xml_pubmed_citations quebrava com AttributeError quando não tem (opcional no JATS), pois presumia que xml_pubmed_title_reference_list já tinha criado <ReferenceList>. Agora cria o elemento se necessário -- válido pela DTD (ReferenceList (Title?, Reference*, ReferenceList*)). 3. get_first_page só olhava elocation-id, nunca fpage; LastPage não existia. Artigos com paginação tradicional (fpage/lpage) perdiam essa informação por completo. Agora FirstPage prefere fpage e cai para elocation-id só quando não há paginação (fluxo de Publicação Contínua); LastPage foi implementado. Validado: 24/26 amostras reais passam na DTD real (as 2 restantes dependem do fix de Author FirstName/LastName de #1236/#1245, ainda não mesclado nesta branch -- não são regressão destes fixes). --- packtools/sps/formats/pubmed.py | 44 +++++++- tests/sps/formats/test_pubmed.py | 183 +++++++++++++++++++++++++++++++ 2 files changed, 224 insertions(+), 3 deletions(-) diff --git a/packtools/sps/formats/pubmed.py b/packtools/sps/formats/pubmed.py index 73f94ea27..c33c32dbf 100644 --- a/packtools/sps/formats/pubmed.py +++ b/packtools/sps/formats/pubmed.py @@ -126,9 +126,15 @@ def xml_pubmed_issue_pipe(xml_pubmed, xml_tree): def get_date(xml_tree): + """ + article_date (epub) alone misses articles whose only pub-date is tagged + date-type="collection"/pub-type="epub-ppub" or "collection" -- common in + older SciELO articles (~80% of tests/samples/*.xml). Same collection_date + fallback already used by references.py and oai_dc.py for this reason. + """ date = dates.ArticleDates(xml_tree) - return date.article_date + return date.article_date or date.collection_date def xml_pubmed_pub_date_pipe(xml_pubmed, xml_tree): @@ -206,9 +212,14 @@ def xml_pubmed_vernacular_title_pipe(xml_pubmed, xml_tree): def get_first_page(xml_tree): + """ + Prefers the real //fpage (traditional pagination); falls back to + //elocation-id for continuous-publication articles that have no fpage + at all -- NCBI's own convention for representing those in FirstPage. + """ issue = front_articlemeta_issue.ArticleMetaIssue(xml_tree) - return issue.elocation_id + return issue.fpage or issue.elocation_id def xml_pubmed_first_page_pipe(xml_pubmed, xml_tree): @@ -223,6 +234,23 @@ def xml_pubmed_first_page_pipe(xml_pubmed, xml_tree): xml_pubmed.append(el) +def get_last_page(xml_tree): + issue = front_articlemeta_issue.ArticleMetaIssue(xml_tree) + + return issue.lpage + + +def xml_pubmed_last_page_pipe(xml_pubmed, xml_tree): + """ + <LastPage>232</LastPage> + """ + last_page = get_last_page(xml_tree) + if last_page is not None: + el = ET.Element("LastPage") + el.text = last_page + xml_pubmed.append(el) + + def get_elocation(xml_tree): ids = article_ids.ArticleIds(xml_tree) @@ -295,6 +323,7 @@ def build_pubmed_article(xml_tree): xml_pubmed_article_title_pipe(xml_pubmed, xml_tree) xml_pubmed_vernacular_title_pipe(xml_pubmed, xml_tree) xml_pubmed_first_page_pipe(xml_pubmed, xml_tree) + xml_pubmed_last_page_pipe(xml_pubmed, xml_tree) xml_pubmed_elocation_pipe(xml_pubmed, xml_tree) xml_pubmed_language_pipe(xml_pubmed, xml_tree) xml_pubmed_author_list(xml_pubmed, xml_tree) @@ -693,8 +722,17 @@ def xml_pubmed_citations(xml_pubmed, xml_tree): </Reference> </ReferenceList> """ - refs = references.XMLReferences(xml_tree).main_references + refs = list(references.XMLReferences(xml_tree).main_references) + if not refs: + return xml = xml_pubmed.find("./ReferenceList") + if xml is None: + # ref-list/title is optional in JATS; xml_pubmed_title_reference_list + # only creates <ReferenceList> when a title is present, but a + # reference list without a <Title> is still valid per the DTD + # (`ReferenceList (Title?, Reference*, ReferenceList*)`). + xml = ET.Element("ReferenceList") + xml_pubmed.append(xml) for ref in refs: ref_el = ET.Element("Reference") citation = ET.Element("Citation") diff --git a/tests/sps/formats/test_pubmed.py b/tests/sps/formats/test_pubmed.py index bd22f51b8..08ea60da2 100644 --- a/tests/sps/formats/test_pubmed.py +++ b/tests/sps/formats/test_pubmed.py @@ -13,6 +13,7 @@ xml_pubmed_pub_date_pipe, xml_pubmed_article_title_pipe, xml_pubmed_first_page_pipe, + xml_pubmed_last_page_pipe, xml_pubmed_elocation_pipe, xml_pubmed_language_pipe, xml_pubmed_author_list, @@ -400,6 +401,48 @@ def test_xml_pubmed_pub_date_pipe(self): self.assertEqual(obtained, expected) + def test_xml_pubmed_pub_date_pipe_falls_back_to_collection_date(self): + # date-type="pub"/pub-type="epub" is absent; the only pub-date is + # pub-type="epub-ppub" (date-type="collection"), common in older + # SciELO articles -- article_date alone misses this, so get_date + # must fall back to collection_date instead of treating it as + # "no date at all". + expected = ( + '<Article>' + '<Journal>' + '<PubDate PubStatus="epublish">' + '<Year>2014</Year>' + '<Month>04</Month>' + '</PubDate>' + '</Journal>' + '</Article>' + ) + xml_pubmed = ET.fromstring( + '<Article>' + '<Journal/>' + '</Article>' + ) + xml_tree = ET.fromstring( + '<article xmlns:mml="http://www.w3.org/1998/Math/MathML" ' + 'xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'article-type="research-article" dtd-version="1.1" specific-use="sps-1.9" xml:lang="en">' + '<front>' + '<article-meta>' + '<pub-date pub-type="epub-ppub">' + '<month>04</month>' + '<year>2014</year>' + '</pub-date>' + '</article-meta>' + '</front>' + '</article>' + ) + + xml_pubmed_pub_date_pipe(xml_pubmed, xml_tree) + + obtained = ET.tostring(xml_pubmed, encoding="utf-8").decode("utf-8") + + self.assertEqual(obtained, expected) + def test_xml_pubmed_pub_date_pipe_without_day(self): expected = ( '<Article>' @@ -706,6 +749,89 @@ def test_xml_pubmed_first_page_pipe_without_first_page(self): self.assertEqual(obtained, expected) + def test_xml_pubmed_first_page_pipe_prefers_real_fpage_over_elocation_id(self): + # Articles with traditional pagination (//fpage) must not lose it -- + # elocation-id is only a fallback for continuous-publication articles + # that have no fpage at all. + expected = ( + '<Article>' + '<FirstPage LZero="save">227</FirstPage>' + '</Article>' + ) + xml_pubmed = ET.fromstring( + '<Article/>' + ) + xml_tree = ET.fromstring( + '<article xmlns:mml="http://www.w3.org/1998/Math/MathML" ' + 'xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'article-type="research-article" dtd-version="1.1" specific-use="sps-1.9" xml:lang="en">' + '<front>' + '<article-meta>' + '<fpage>227</fpage>' + '<elocation-id>e2022440</elocation-id>' + '</article-meta>' + '</front>' + '</article>' + ) + + xml_pubmed_first_page_pipe(xml_pubmed, xml_tree) + + obtained = ET.tostring(xml_pubmed, encoding="utf-8").decode("utf-8") + + self.assertEqual(obtained, expected) + + def test_xml_pubmed_last_page_pipe(self): + expected = ( + '<Article>' + '<LastPage>232</LastPage>' + '</Article>' + ) + xml_pubmed = ET.fromstring( + '<Article/>' + ) + xml_tree = ET.fromstring( + '<article xmlns:mml="http://www.w3.org/1998/Math/MathML" ' + 'xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'article-type="research-article" dtd-version="1.1" specific-use="sps-1.9" xml:lang="en">' + '<front>' + '<article-meta>' + '<fpage>227</fpage>' + '<lpage>232</lpage>' + '</article-meta>' + '</front>' + '</article>' + ) + + xml_pubmed_last_page_pipe(xml_pubmed, xml_tree) + + obtained = ET.tostring(xml_pubmed, encoding="utf-8").decode("utf-8") + + self.assertEqual(obtained, expected) + + def test_xml_pubmed_last_page_pipe_without_last_page(self): + expected = ( + '<Article/>' + ) + xml_pubmed = ET.fromstring( + '<Article/>' + ) + xml_tree = ET.fromstring( + '<article xmlns:mml="http://www.w3.org/1998/Math/MathML" ' + 'xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'article-type="research-article" dtd-version="1.1" specific-use="sps-1.9" xml:lang="en">' + '<front>' + '<article-meta>' + '</article-meta>' + '</front>' + '</article>' + ) + + xml_pubmed_last_page_pipe(xml_pubmed, xml_tree) + + obtained = ET.tostring(xml_pubmed, encoding="utf-8").decode("utf-8") + + self.assertEqual(obtained, expected) + def test_xml_pubmed_elocation_pipe(self): expected = ( '<Article>' @@ -1590,6 +1716,63 @@ def test_xml_pubmed_citations_without_article_list_id(self): self.assertEqual(obtained, expected) + def test_xml_pubmed_citations_creates_reference_list_when_ref_list_has_no_title(self): + # ref-list/title is optional in JATS. xml_pubmed_title_reference_list + # only creates <ReferenceList> when a title is present, so + # xml_pubmed_citations must create it itself when there are + # references but no title -- it used to crash with AttributeError + # ('NoneType' object has no attribute 'append') in this case. + expected = ( + '<Article>' + '<ReferenceList>' + '<Reference>' + '<Citation>Some citation text.</Citation>' + '</Reference>' + '</ReferenceList>' + '</Article>' + ) + xml_pubmed = ET.fromstring( + '<Article/>' + ) + xml_tree = ET.fromstring( + '<article xmlns:mml="http://www.w3.org/1998/Math/MathML" ' + 'xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'article-type="research-article" dtd-version="1.1" specific-use="sps-1.9" xml:lang="en">' + '<back>' + '<ref-list>' + '<ref id="B1">' + '<mixed-citation>Some citation text.</mixed-citation>' + '</ref>' + '</ref-list>' + '</back>' + '</article>' + ) + + xml_pubmed_citations(xml_pubmed, xml_tree) + + obtained = ET.tostring(xml_pubmed, encoding="utf-8").decode("utf-8") + + self.assertEqual(obtained, expected) + + def test_xml_pubmed_citations_without_refs_does_not_create_reference_list(self): + expected = '<Article/>' + xml_pubmed = ET.fromstring( + '<Article/>' + ) + xml_tree = ET.fromstring( + '<article xmlns:mml="http://www.w3.org/1998/Math/MathML" ' + 'xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'article-type="research-article" dtd-version="1.1" specific-use="sps-1.9" xml:lang="en">' + '<back></back>' + '</article>' + ) + + xml_pubmed_citations(xml_pubmed, xml_tree) + + obtained = ET.tostring(xml_pubmed, encoding="utf-8").decode("utf-8") + + self.assertEqual(obtained, expected) + def test_xml_pubmed_abstract_without_sections(self): self.maxDiff = None expected = ( From 64147375e9d5ae8ed52d50ef7989e690c012bde5 Mon Sep 17 00:00:00 2001 From: Rossi-Luciano <luciano.rossi.lucross@gmail.com> Date: Fri, 10 Jul 2026 15:40:24 -0300 Subject: [PATCH 5/5] =?UTF-8?q?test(pubmed):=20su=C3=ADte=20de=20integra?= =?UTF-8?q?=C3=A7=C3=A3o=20validando=20as=2026=20amostras=20reais=20contra?= =?UTF-8?q?=20a=20DTD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Diferente de test_pubmed.py (uma pipe por vez, XML sintético mínimo), este teste roda o pipeline completo (build_pubmed_article + build_article_set_xml) em cada amostra real de tests/samples/*.xml e valida o resultado contra a DTD real do PubMed. A DTD foi vendorizada em tests/fixtures/pubmed/PubMed.dtd para o teste não depender de rede (CI sem acesso à internet, ou instabilidade do servidor da NLM não devem quebrar a suíte). Essa é exatamente a lacuna que deixou passar despercebidos os 3 bugs corrigidos no commit anterior (PubDate/collection_date, crash em citations, FirstPage/LastPage) por 8 PRs já abertos. Duas amostras (example.xml, 0034-8910-rsp-48-2-0249.xml) ainda dependem de fixes que vivem em #1236/PR #1245 (Author FirstName/ LastName pairing e fallback `or []` em get_affiliations), ainda não mesclados nesta branch -- marcadas em KNOWN_FAILURES com asserção inversa, para que a suíte avise (ao invés de silenciosamente continuar verde) quando alguém puder remover a entrada. --- tests/fixtures/pubmed/PubMed.dtd | 222 +++++++++++++++++++ tests/sps/formats/test_pubmed_integration.py | 91 ++++++++ 2 files changed, 313 insertions(+) create mode 100644 tests/fixtures/pubmed/PubMed.dtd create mode 100644 tests/sps/formats/test_pubmed_integration.py diff --git a/tests/fixtures/pubmed/PubMed.dtd b/tests/fixtures/pubmed/PubMed.dtd new file mode 100644 index 000000000..15750070f --- /dev/null +++ b/tests/fixtures/pubmed/PubMed.dtd @@ -0,0 +1,222 @@ +<!-- PubMed Journal Article DTD Version 3.0 --> + +<!-- This document is still under revision and may change. + + Changes: + + version 2.8 + - Added elements for reference citations: + ReferenceList + Title + Reference + Citation + PMID + - Added CDATA Type attribute to OtherAbstract + + version 2.7 + - Added MathML3 support files + - removed directly referenced character files (ISO*.ent) in favor of MathML characters + - added pubmedchars/ISOchars.ent to define 138 ISO characters that were defined + in the previous PubMed.dtd but were not covered by the mathml iso sets + + version 2.6: + - added attributes on Article tag: + VersionID + VersionDate + + version 2.5: + - added tags: + OtherAbstract + + version 2.4: + - added tags: + CopyrightInformation + + + version 2.3: + - added tags: + GroupList + Group + GroupName + IndividualName + ELocationID + + version 2.2: + - added "EmptyYN" attribute to FirstName tag + + version 2.1: + - AuthorList changed from containing (Author+) to (Author*) + - Removal of FullTextURL element + - Addition of euro entity + + --> + +<!-- Typical usage: + + <!DOCTYPE ArticleSet PUBLIC "-//NLM//DTD PubMed 2.0//EN" "http://www.ncbi.nlm.nih.gov:80/entrez/query/static/PubMed.dtd"> + <ArticleSet> + ... + </ArticleSet> + + or: + + <!DOCTYPE ArticleSet PUBLIC "-//NLM//DTD PubMed//EN" "http://www.ncbi.nlm.nih.gov:80/entrez/query/static/PubMed.dtd"> + <ArticleSet> + ... + </ArticleSet> + --> + + + +<!-- ============================================================= --> +<!-- MATHML 3.0 SETUP --> +<!-- ============================================================= --> +<!-- MATHML SETUP FILE --> +<!ENTITY % mathml-in-pubmed SYSTEM "mathml-in-pubmed.mod" > +%mathml-in-pubmed; + +<!-- ============================================================= --> +<!-- REMAINING PUBMED CHARACTERS --> +<!-- ============================================================= --> +<!ENTITY % ISOchars SYSTEM "pubmedchars/ISOchars.ent"> +%ISOchars; + + +<!ENTITY % data "#PCDATA | sup | inf | b | u | i | sub"> + +<!ENTITY % pub.status.int "pmc | pmcr | pubmed | pubmedr | + premedline | medline | medliner"> +<!ENTITY % pub.status "(received | accepted | epublish | + ppublish | revised | aheadofprint | ecollection | + %pub.status.int;)"> + +<!ENTITY % art.id.type.int "pubmed | medline | pmcid"> +<!ENTITY % art.id.type "(doi | pii | pmcpid | pmpid | %art.id.type.int;)"> + + +<!-- This is the top level element --> +<!ELEMENT ArticleSet (Article+)> +<!ATTLIST ArticleSet > + +<!ELEMENT Article (Journal, Replaces?, + ArticleTitle?, VernacularTitle?, + FirstPage?, LastPage?, ELocationID*, + Language*, AuthorList?, GroupList?, + PublicationType*, ArticleIdList?, + History?, Abstract?, OtherAbstract*, CopyrightInformation?, CoiStatement?, ObjectList?, ReferenceList*, ArchiveCopySource? )> +<!ATTLIST Article + VersionID CDATA #IMPLIED + VersionDate CDATA #IMPLIED + > + +<!ELEMENT AbstractText (%data; | mml:math | DispFormula)*> +<!ATTLIST AbstractText Label CDATA #REQUIRED> + +<!ENTITY % abstract.common "#PCDATA | AbstractText | sup | inf | b | u | i | sub | mml:math | DispFormula"> +<!ENTITY % abstract "(%abstract.common;)*"> +<!ENTITY % other.abstract "(%abstract.common; | ArticleTitle)*"> + +<!ELEMENT Journal (PublisherName, + JournalTitle, Issn, + Volume?, Issue?, + PubDate)> +<!ELEMENT PublisherName (#PCDATA)> +<!ELEMENT JournalTitle (#PCDATA)> +<!ELEMENT Issn (#PCDATA)> +<!ELEMENT Volume (#PCDATA)> +<!ELEMENT Issue (#PCDATA)> +<!ELEMENT PubDate (Year, Month?, Day?, Season?)> +<!ATTLIST PubDate + PubStatus %pub.status; "ppublish" > + +<!ELEMENT Year (#PCDATA)> +<!ELEMENT Month (#PCDATA)> +<!ELEMENT Day (#PCDATA)> +<!ELEMENT Season (#PCDATA)> +<!-- End of PubDate group --> +<!ELEMENT History (PubDate*)> +<!-- End of Journal group --> +<!ELEMENT Replaces (#PCDATA)> +<!ATTLIST Replaces + IdType %art.id.type; "pubmed" > +<!ELEMENT ArticleTitle (%data; | mml:math)*> +<!ELEMENT VernacularTitle (%data; | mml:math)*> +<!ELEMENT FirstPage (#PCDATA)> +<!ATTLIST FirstPage + LZero (Save|save|delete) "delete" > +<!ELEMENT LastPage (#PCDATA)> +<!ELEMENT Language (#PCDATA)> +<!ELEMENT AuthorList (Author*)> +<!ENTITY % affiliation "(Affiliation|AffiliationInfo+)?"> +<!ELEMENT Author (((FirstName, MiddleName?, LastName, Suffix?, Initials?)| + CollectiveName), %affiliation;, Identifier*)> +<!ATTLIST Author + EqualContrib (Y|N) #IMPLIED > +<!ELEMENT FirstName (#PCDATA)> +<!ATTLIST FirstName + EmptyYN (Y|N) "N"> +<!ELEMENT MiddleName (#PCDATA)> +<!ELEMENT LastName (#PCDATA)> +<!ELEMENT CollectiveName (#PCDATA)> +<!ELEMENT Suffix (#PCDATA)> +<!ELEMENT Initials (#PCDATA)> +<!ELEMENT Affiliation (#PCDATA)> +<!ELEMENT AffiliationInfo (Affiliation,Identifier*)> +<!ELEMENT Identifier (#PCDATA)> +<!ATTLIST Identifier + Source CDATA #REQUIRED> +<!-- End of Author group --> +<!-- End of AuthorList group --> +<!ELEMENT PublicationType (#PCDATA)> +<!ELEMENT ArticleIdList (ArticleId*)> +<!ELEMENT ArticleId (#PCDATA)> +<!ATTLIST ArticleId + IdType %art.id.type; "pii" > + +<!ELEMENT Abstract %abstract; > +<!ELEMENT OtherAbstract %other.abstract;> +<!ATTLIST OtherAbstract + Language CDATA #REQUIRED + Type CDATA #IMPLIED> + +<!ELEMENT ObjectList (Object)+> +<!ELEMENT Object (Param)*> +<!ATTLIST Object Type CDATA #REQUIRED> +<!ELEMENT Param (%data; | mml:math)*> +<!ATTLIST Param Name CDATA #REQUIRED> + +<!ELEMENT ReferenceList (Title?, Reference*, ReferenceList*) > +<!ELEMENT Title (#PCDATA)> +<!ELEMENT Reference ((Citation | PMID), ArticleIdList?) > +<!ELEMENT Citation (%data; | mml:math)*> +<!ELEMENT PMID (#PCDATA) > +<!ATTLIST PMID + VersionID CDATA #IMPLIED > + +<!-- End of Article group --> + +<!-- End of ArticleSet group --> + +<!ELEMENT sup (%data;)*> +<!ELEMENT inf (%data;)*> +<!ELEMENT sub (%data;)*> +<!ELEMENT b (%data;)*> +<!ELEMENT i (%data;)*> +<!ELEMENT u (%data;)*> + +<!ELEMENT DispFormula (mml:math) > + +<!ELEMENT GroupList (Group+)> +<!ELEMENT Group (GroupName?,IndividualName+)> +<!ELEMENT GroupName (#PCDATA)> +<!ELEMENT IndividualName (FirstName, MiddleName?, LastName, Suffix?, %affiliation; , Identifier*)> + +<!ELEMENT ELocationID (#PCDATA)> +<!ATTLIST ELocationID + EIdType (doi | pii) #REQUIRED + ValidYN (Y | N) "Y" + > +<!ELEMENT CopyrightInformation (#PCDATA)> +<!ELEMENT CoiStatement (%data;)* > +<!ELEMENT ArchiveCopySource (#PCDATA)> +<!ATTLIST ArchiveCopySource DocType CDATA #FIXED "pdf"> diff --git a/tests/sps/formats/test_pubmed_integration.py b/tests/sps/formats/test_pubmed_integration.py new file mode 100644 index 000000000..9e2c70f43 --- /dev/null +++ b/tests/sps/formats/test_pubmed_integration.py @@ -0,0 +1,91 @@ +import glob +import os +import unittest + +from lxml import etree as ET + +from packtools.sps.formats import pubmed + +HERE = os.path.dirname(os.path.abspath(__file__)) +DTD_PATH = os.path.join(HERE, "..", "..", "fixtures", "pubmed", "PubMed.dtd") +SAMPLES_DIR = os.path.join(HERE, "..", "..", "samples") + +# Samples that don't produce DTD-valid PubMed XML yet on this branch because +# the fix lives in an independent, not-yet-merged sub-issue PR of #1226. +# When one of these unexpectedly starts passing, `assertRaises` below will +# fail loudly -- that's the signal to remove the corresponding entry. +KNOWN_FAILURES = { + "example.xml": ( + "Author sem <surname> (só <given-names>) produz Author sem " + "LastName -- corrigido em #1236/PR #1245, ainda não mesclado aqui." + ), + "0034-8910-rsp-48-2-0249.xml": ( + "contrib sem xref de afiliação faz get_affiliations quebrar com " + "TypeError -- corrigido com `or []` em #1236/PR #1245, ainda não " + "mesclado aqui." + ), +} + + +def _sample_paths(): + return sorted(glob.glob(os.path.join(SAMPLES_DIR, "*.xml"))) + + +def _build_article_set_xml(path): + xml_tree = ET.parse(path).getroot() + article = pubmed.build_pubmed_article(xml_tree) + return pubmed.build_article_set_xml([article]) + + +class PubmedRealSamplesEndToEndTest(unittest.TestCase): + """ + Testes de integração fim-a-fim: cada amostra real em tests/samples/ + passa pelo pipeline completo (build_pubmed_article + build_article_set_xml) + e o resultado é validado contra a DTD real do PubMed (vendorizada em + tests/fixtures/pubmed/PubMed.dtd, para não depender de rede). Diferente + de test_pubmed.py, que testa cada pipe isoladamente com XML sintético + mínimo, aqui o objetivo é pegar regressões que só aparecem com artigos + reais e completos (datas, paginação, referências, etc. combinados). + """ + + @classmethod + def setUpClass(cls): + cls.dtd = ET.DTD(DTD_PATH) + + def _assert_produces_valid_pubmed_xml(self, path): + xml_set = _build_article_set_xml(path) + doc = ET.fromstring(xml_set.encode("utf-8")) + valid = self.dtd.validate(doc) + if not valid: + self.fail(str(self.dtd.error_log.filter_from_errors())) + + def test_real_samples_produce_dtd_valid_pubmed_xml(self): + paths = _sample_paths() + self.assertTrue(paths, "nenhuma amostra encontrada em tests/samples/*.xml") + + for path in paths: + name = os.path.basename(path) + with self.subTest(sample=name): + if name in KNOWN_FAILURES: + with self.assertRaises( + (AssertionError, TypeError), + msg=( + f"{name} passou a gerar XML válido -- remova a " + f"entrada em KNOWN_FAILURES ({KNOWN_FAILURES[name]})" + ), + ): + self._assert_produces_valid_pubmed_xml(path) + else: + self._assert_produces_valid_pubmed_xml(path) + + def test_no_stale_known_failures(self): + """Garante que KNOWN_FAILURES só referencia amostras que existem.""" + existing = {os.path.basename(p) for p in _sample_paths()} + stale = set(KNOWN_FAILURES) - existing + self.assertFalse( + stale, f"KNOWN_FAILURES referencia amostras inexistentes: {stale}" + ) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file