diff --git a/packtools/sps/formats/pubmed.py b/packtools/sps/formats/pubmed.py index f99b2651d..c33c32dbf 100644 --- a/packtools/sps/formats/pubmed.py +++ b/packtools/sps/formats/pubmed.py @@ -16,6 +16,19 @@ ) +class MissingRequiredElementError(Exception): + """Raised when a SciELO article has no data for an element the PubMed + DTD marks as required (no `?`), so a valid
cannot be built + for it. `element_path` identifies which element (e.g. "Journal/PubDate").""" + + def __init__(self, element_path): + self.element_path = element_path + super().__init__( + f"{element_path} é obrigatório na DTD do PubMed e não foi " + "possível determinar seu valor para este artigo." + ) + + def xml_pubmed_article_pipe(): return ET.Element("Article") @@ -113,9 +126,15 @@ def xml_pubmed_issue_pipe(xml_pubmed, xml_tree): def get_date(xml_tree): + """ + article_date (epub) alone misses articles whose only pub-date is tagged + date-type="collection"/pub-type="epub-ppub" or "collection" -- common in + older SciELO articles (~80% of tests/samples/*.xml). Same collection_date + fallback already used by references.py and oai_dc.py for this reason. + """ date = dates.ArticleDates(xml_tree) - return date.article_date + return date.article_date or date.collection_date def xml_pubmed_pub_date_pipe(xml_pubmed, xml_tree): @@ -127,20 +146,21 @@ def xml_pubmed_pub_date_pipe(xml_pubmed, xml_tree): """ date = get_date(xml_tree) - if date is not None: - dt = ET.Element("PubDate") - dt.set("PubStatus", "epublish") - for element in ["year", "month", "day"]: - # TODO - # Season - # The season of publication. e.g.,Winter, Spring, Summer, Fall. Do not use if a Month is available. - # There is no example of using this value in the files. - - if date.get(element): - el = ET.Element(element.capitalize()) - el.text = date.get(element) - dt.append(el) - xml_pubmed.find("Journal").append(dt) + if date is None: + raise MissingRequiredElementError("Journal/PubDate") + dt = ET.Element("PubDate") + dt.set("PubStatus", "epublish") + for element in ["year", "month", "day"]: + # TODO + # Season + # The season of publication. e.g.,Winter, Spring, Summer, Fall. Do not use if a Month is available. + # There is no example of using this value in the files. + + if date.get(element): + el = ET.Element(element.capitalize()) + el.text = date.get(element) + dt.append(el) + xml_pubmed.find("Journal").append(dt) def xml_pubmed_replaces_pipe(xml_pubmed, xml_tree): @@ -192,9 +212,14 @@ def xml_pubmed_vernacular_title_pipe(xml_pubmed, xml_tree): def get_first_page(xml_tree): + """ + Prefers the real //fpage (traditional pagination); falls back to + //elocation-id for continuous-publication articles that have no fpage + at all -- NCBI's own convention for representing those in FirstPage. + """ issue = front_articlemeta_issue.ArticleMetaIssue(xml_tree) - return issue.elocation_id + return issue.fpage or issue.elocation_id def xml_pubmed_first_page_pipe(xml_pubmed, xml_tree): @@ -209,6 +234,23 @@ def xml_pubmed_first_page_pipe(xml_pubmed, xml_tree): xml_pubmed.append(el) +def get_last_page(xml_tree): + issue = front_articlemeta_issue.ArticleMetaIssue(xml_tree) + + return issue.lpage + + +def xml_pubmed_last_page_pipe(xml_pubmed, xml_tree): + """ + 232 + """ + last_page = get_last_page(xml_tree) + if last_page is not None: + el = ET.Element("LastPage") + el.text = last_page + xml_pubmed.append(el) + + def get_elocation(xml_tree): ids = article_ids.ArticleIds(xml_tree) @@ -256,7 +298,20 @@ def xml_pubmed_language_pipe(xml_pubmed, xml_tree): add_langs(xml_pubmed, xml_tree) -def pipeline_pubmed(xml_tree, pretty_print=True): +PUBMED_DOCTYPE = ( + '' +) + + +def xml_pubmed_article_set_pipe(): + return ET.Element("ArticleSet") + + +def build_pubmed_article(xml_tree): + """ + Builds a single
element for one SciELO XML tree. + """ xml_pubmed = xml_pubmed_article_pipe() xml_pubmed_journal_pipe(xml_pubmed) xml_pubmed_publisher_name_pipe(xml_pubmed, xml_tree) @@ -266,23 +321,65 @@ def pipeline_pubmed(xml_tree, pretty_print=True): xml_pubmed_issue_pipe(xml_pubmed, xml_tree) xml_pubmed_pub_date_pipe(xml_pubmed, xml_tree) xml_pubmed_article_title_pipe(xml_pubmed, xml_tree) + xml_pubmed_vernacular_title_pipe(xml_pubmed, xml_tree) xml_pubmed_first_page_pipe(xml_pubmed, xml_tree) + xml_pubmed_last_page_pipe(xml_pubmed, xml_tree) xml_pubmed_elocation_pipe(xml_pubmed, xml_tree) xml_pubmed_language_pipe(xml_pubmed, xml_tree) xml_pubmed_author_list(xml_pubmed, xml_tree) xml_pubmed_publication_type(xml_pubmed, xml_tree) xml_pubmed_article_id(xml_pubmed, xml_tree) xml_pubmed_history(xml_pubmed, xml_tree) + xml_pubmed_abstract(xml_pubmed, xml_tree) + xml_pubmed_other_abstract(xml_pubmed, xml_tree) xml_pubmed_copyright_information(xml_pubmed, xml_tree) xml_pubmed_coi_statement(xml_pubmed, xml_tree) xml_pubmed_object_list(xml_pubmed, xml_tree) xml_pubmed_title_reference_list(xml_pubmed, xml_tree) xml_pubmed_citations(xml_pubmed, xml_tree) - xml_pubmed_abstract(xml_pubmed, xml_tree) - xml_pubmed_other_abstract(xml_pubmed, xml_tree) + return xml_pubmed + + +def build_article_set_xml(articles, pretty_print=True): + """ + Wraps a list of already-built
elements (see build_pubmed_article) + into the complete PubMed XML document ( + + ). + """ + article_set = xml_pubmed_article_set_pipe() + for article in articles: + article_set.append(article) - xml_tree = ET.ElementTree(xml_pubmed) - return ET.tostring(xml_tree, pretty_print=pretty_print, encoding="utf-8").decode("utf-8") + return ET.tostring( + ET.ElementTree(article_set), + pretty_print=pretty_print, + encoding="utf-8", + xml_declaration=True, + doctype=PUBMED_DOCTYPE, + ).decode("utf-8") + + +def pipeline_pubmed_set(xml_trees, pretty_print=True): + """ + Builds a complete PubMed XML document ( + + ) containing one
per SciELO XML tree in xml_trees. + + Raises MissingRequiredElementError immediately if any xml_tree is + missing data for a DTD-required element (e.g. no usable publication + date). Callers that want to skip such articles + instead of failing the whole batch (e.g. the batch CLI) should build + each
themselves via build_pubmed_article and assemble the + final document with build_article_set_xml. + """ + articles = [build_pubmed_article(xml_tree) for xml_tree in xml_trees] + return build_article_set_xml(articles, pretty_print=pretty_print) + + +def pipeline_pubmed(xml_tree, pretty_print=True): + """ + Builds a complete PubMed XML document for a single SciELO XML tree. + """ + return pipeline_pubmed_set([xml_tree], pretty_print=pretty_print) def get_authors(xml_tree): @@ -625,8 +722,17 @@ def xml_pubmed_citations(xml_pubmed, xml_tree): """ - refs = references.XMLReferences(xml_tree).main_references + refs = list(references.XMLReferences(xml_tree).main_references) + if not refs: + return xml = xml_pubmed.find("./ReferenceList") + if xml is None: + # ref-list/title is optional in JATS; xml_pubmed_title_reference_list + # only creates when a title is present, but a + # reference list without a is still valid per the DTD + # (`ReferenceList (Title?, Reference*, ReferenceList*)`). + xml = ET.Element("ReferenceList") + xml_pubmed.append(xml) for ref in refs: ref_el = ET.Element("Reference") citation = ET.Element("Citation") diff --git a/packtools/sps/formats/pubmed_generator.py b/packtools/sps/formats/pubmed_generator.py index eccd7388c..f626775a1 100644 --- a/packtools/sps/formats/pubmed_generator.py +++ b/packtools/sps/formats/pubmed_generator.py @@ -1,12 +1,57 @@ import argparse +import sys from packtools.sps.formats import pubmed -from packtools.sps.utils import xml_utils +from packtools.sps.pid_provider import xml_sps_lib + + +def get_xml_trees_and_errors(path_to_read): + """ + Reads a single SciELO .xml file or every XML file inside a .zip package + (xml_sps_lib.get_xml_items handles both transparently) and returns: + - the list of parsed xmltree objects, ready for build_articles_and_errors + - the list of per-file errors encountered along the way (dicts with + "filename"/"error"/"type_error" keys), so a single bad file in a + package doesn't stop the whole batch. + """ + xml_trees = [] + errors = [] + for item in xml_sps_lib.get_xml_items(path_to_read, capture_errors=True): + xml_with_pre = item.get("xml_with_pre") + if xml_with_pre is None: + errors.append(item) + continue + xml_trees.append(xml_with_pre.xmltree) + return xml_trees, errors + + +def build_articles_and_errors(xml_trees): + """ + Builds one <Article> per xml_tree, skipping (with an error record) any + article whose SciELO XML is missing data for a PubMed DTD-required + element (e.g. no usable publication date), since such an article can't + be represented, but that alone shouldn't fail the rest of the batch. + """ + articles = [] + errors = [] + for xml_tree in xml_trees: + try: + articles.append(pubmed.build_pubmed_article(xml_tree)) + except pubmed.MissingRequiredElementError as exc: + ids = pubmed.get_elocation(xml_tree) + identifier = ids.get("doi") or ids.get("v2") or "artigo sem identificador" + errors.append({"filename": identifier, "error": str(exc)}) + return articles, errors def main(): parser = argparse.ArgumentParser( - description="Convert XML file from SciELO format to Pubmed format." + description=( + "Convert one or more XML files from SciELO format to PubMed format. " + "Accepts a single .xml file or a .zip package containing multiple " + "XML files; every article found is written as one <Article> inside " + "a single <ArticleSet> output document." + ) ) parser.add_argument( "-i", @@ -14,7 +59,7 @@ def main(): action="store", dest="path_to_read", required=True, - help="Path for reading the SciELO XML file.", + help="Path for reading the SciELO XML file or .zip package.", ) parser.add_argument( "-o", @@ -26,11 +71,34 @@ def main(): ) arguments = parser.parse_args() - xml_tree = xml_utils.get_xml_tree(arguments.path_to_read) - xml_pubmed = pubmed.pipeline_pubmed(xml_tree) + xml_trees, errors = get_xml_trees_and_errors(arguments.path_to_read) + articles, pubdate_errors = build_articles_and_errors(xml_trees) + errors = errors + pubdate_errors + + for error in errors: + print( + "Aviso: falha ao processar {}: {}".format( + error.get("filename", arguments.path_to_read), error.get("error") + ), + file=sys.stderr, + ) + + if not articles: + print( + f"Nenhum artigo válido encontrado em: {arguments.path_to_read}", + file=sys.stderr, + ) + raise SystemExit(1) + + xml_pubmed_set = pubmed.build_article_set_xml(articles) with open(arguments.path_to_write, "w", encoding="utf-8") as file: - file.write(xml_pubmed) - print(f"Arquivo criado em: {arguments.path_to_write}") + file.write(xml_pubmed_set) + + print( + "Arquivo criado em: {} ({} artigo(s), {} erro(s))".format( + arguments.path_to_write, len(articles), len(errors) + ) + ) if __name__ == "__main__": diff --git a/setup.py b/setup.py index 880f6a9c8..e00f6eeff 100644 --- a/setup.py +++ b/setup.py @@ -85,6 +85,7 @@ "package_optimiser=packtools.package_optimiser:main", "package_maker=packtools.package_maker:main", "pdf_generator=packtools.sps.formats.pdf_generator:main", + "pubmed_generator=packtools.sps.formats.pubmed_generator:main", ] } ) diff --git a/tests/fixtures/pubmed/PubMed.dtd b/tests/fixtures/pubmed/PubMed.dtd new file mode 100644 index 000000000..15750070f --- /dev/null +++ b/tests/fixtures/pubmed/PubMed.dtd @@ -0,0 +1,222 @@ +<!-- PubMed Journal Article DTD Version 3.0 --> + +<!-- This document is still under revision and may change. + + Changes: + + version 2.8 + - Added elements for reference citations: + ReferenceList + Title + Reference + Citation + PMID + - Added CDATA Type attribute to OtherAbstract + + version 2.7 + - Added MathML3 support files + - removed directly referenced character files (ISO*.ent) in favor of MathML characters + - added pubmedchars/ISOchars.ent to define 138 ISO characters that were defined + in the previous PubMed.dtd but were not covered by the mathml iso sets + + version 2.6: + - added attributes on Article tag: + VersionID + VersionDate + + version 2.5: + - added tags: + OtherAbstract + + version 2.4: + - added tags: + CopyrightInformation + + + version 2.3: + - added tags: + GroupList + Group + GroupName + IndividualName + ELocationID + + version 2.2: + - added "EmptyYN" attribute to FirstName tag + + version 2.1: + - AuthorList changed from containing (Author+) to (Author*) + - Removal of FullTextURL element + - Addition of euro entity + + --> + +<!-- Typical usage: + + <!DOCTYPE ArticleSet PUBLIC "-//NLM//DTD PubMed 2.0//EN" "http://www.ncbi.nlm.nih.gov:80/entrez/query/static/PubMed.dtd"> + <ArticleSet> + ... + </ArticleSet> + + or: + + <!DOCTYPE ArticleSet PUBLIC "-//NLM//DTD PubMed//EN" "http://www.ncbi.nlm.nih.gov:80/entrez/query/static/PubMed.dtd"> + <ArticleSet> + ... + </ArticleSet> + --> + + + +<!-- ============================================================= --> +<!-- MATHML 3.0 SETUP --> +<!-- ============================================================= --> +<!-- MATHML SETUP FILE --> +<!ENTITY % mathml-in-pubmed SYSTEM "mathml-in-pubmed.mod" > +%mathml-in-pubmed; + +<!-- ============================================================= --> +<!-- REMAINING PUBMED CHARACTERS --> +<!-- ============================================================= --> +<!ENTITY % ISOchars SYSTEM "pubmedchars/ISOchars.ent"> +%ISOchars; + + +<!ENTITY % data "#PCDATA | sup | inf | b | u | i | sub"> + +<!ENTITY % pub.status.int "pmc | pmcr | pubmed | pubmedr | + premedline | medline | medliner"> +<!ENTITY % pub.status "(received | accepted | epublish | + ppublish | revised | aheadofprint | ecollection | + %pub.status.int;)"> + +<!ENTITY % art.id.type.int "pubmed | medline | pmcid"> +<!ENTITY % art.id.type "(doi | pii | pmcpid | pmpid | %art.id.type.int;)"> + + +<!-- This is the top level element --> +<!ELEMENT ArticleSet (Article+)> +<!ATTLIST ArticleSet > + +<!ELEMENT Article (Journal, Replaces?, + ArticleTitle?, VernacularTitle?, + FirstPage?, LastPage?, ELocationID*, + Language*, AuthorList?, GroupList?, + PublicationType*, ArticleIdList?, + History?, Abstract?, OtherAbstract*, CopyrightInformation?, CoiStatement?, ObjectList?, ReferenceList*, ArchiveCopySource? )> +<!ATTLIST Article + VersionID CDATA #IMPLIED + VersionDate CDATA #IMPLIED + > + +<!ELEMENT AbstractText (%data; | mml:math | DispFormula)*> +<!ATTLIST AbstractText Label CDATA #REQUIRED> + +<!ENTITY % abstract.common "#PCDATA | AbstractText | sup | inf | b | u | i | sub | mml:math | DispFormula"> +<!ENTITY % abstract "(%abstract.common;)*"> +<!ENTITY % other.abstract "(%abstract.common; | ArticleTitle)*"> + +<!ELEMENT Journal (PublisherName, + JournalTitle, Issn, + Volume?, Issue?, + PubDate)> +<!ELEMENT PublisherName (#PCDATA)> +<!ELEMENT JournalTitle (#PCDATA)> +<!ELEMENT Issn (#PCDATA)> +<!ELEMENT Volume (#PCDATA)> +<!ELEMENT Issue (#PCDATA)> +<!ELEMENT PubDate (Year, Month?, Day?, Season?)> +<!ATTLIST PubDate + PubStatus %pub.status; "ppublish" > + +<!ELEMENT Year (#PCDATA)> +<!ELEMENT Month (#PCDATA)> +<!ELEMENT Day (#PCDATA)> +<!ELEMENT Season (#PCDATA)> +<!-- End of PubDate group --> +<!ELEMENT History (PubDate*)> +<!-- End of Journal group --> +<!ELEMENT Replaces (#PCDATA)> +<!ATTLIST Replaces + IdType %art.id.type; "pubmed" > +<!ELEMENT ArticleTitle (%data; | mml:math)*> +<!ELEMENT VernacularTitle (%data; | mml:math)*> +<!ELEMENT FirstPage (#PCDATA)> +<!ATTLIST FirstPage + LZero (Save|save|delete) "delete" > +<!ELEMENT LastPage (#PCDATA)> +<!ELEMENT Language (#PCDATA)> +<!ELEMENT AuthorList (Author*)> +<!ENTITY % affiliation "(Affiliation|AffiliationInfo+)?"> +<!ELEMENT Author (((FirstName, MiddleName?, LastName, Suffix?, Initials?)| + CollectiveName), %affiliation;, Identifier*)> +<!ATTLIST Author + EqualContrib (Y|N) #IMPLIED > +<!ELEMENT FirstName (#PCDATA)> +<!ATTLIST FirstName + EmptyYN (Y|N) "N"> +<!ELEMENT MiddleName (#PCDATA)> +<!ELEMENT LastName (#PCDATA)> +<!ELEMENT CollectiveName (#PCDATA)> +<!ELEMENT Suffix (#PCDATA)> +<!ELEMENT Initials (#PCDATA)> +<!ELEMENT Affiliation (#PCDATA)> +<!ELEMENT AffiliationInfo (Affiliation,Identifier*)> +<!ELEMENT Identifier (#PCDATA)> +<!ATTLIST Identifier + Source CDATA #REQUIRED> +<!-- End of Author group --> +<!-- End of AuthorList group --> +<!ELEMENT PublicationType (#PCDATA)> +<!ELEMENT ArticleIdList (ArticleId*)> +<!ELEMENT ArticleId (#PCDATA)> +<!ATTLIST ArticleId + IdType %art.id.type; "pii" > + +<!ELEMENT Abstract %abstract; > +<!ELEMENT OtherAbstract %other.abstract;> +<!ATTLIST OtherAbstract + Language CDATA #REQUIRED + Type CDATA #IMPLIED> + +<!ELEMENT ObjectList (Object)+> +<!ELEMENT Object (Param)*> +<!ATTLIST Object Type CDATA #REQUIRED> +<!ELEMENT Param (%data; | mml:math)*> +<!ATTLIST Param Name CDATA #REQUIRED> + +<!ELEMENT ReferenceList (Title?, Reference*, ReferenceList*) > +<!ELEMENT Title (#PCDATA)> +<!ELEMENT Reference ((Citation | PMID), ArticleIdList?) > +<!ELEMENT Citation (%data; | mml:math)*> +<!ELEMENT PMID (#PCDATA) > +<!ATTLIST PMID + VersionID CDATA #IMPLIED > + +<!-- End of Article group --> + +<!-- End of ArticleSet group --> + +<!ELEMENT sup (%data;)*> +<!ELEMENT inf (%data;)*> +<!ELEMENT sub (%data;)*> +<!ELEMENT b (%data;)*> +<!ELEMENT i (%data;)*> +<!ELEMENT u (%data;)*> + +<!ELEMENT DispFormula (mml:math) > + +<!ELEMENT GroupList (Group+)> +<!ELEMENT Group (GroupName?,IndividualName+)> +<!ELEMENT GroupName (#PCDATA)> +<!ELEMENT IndividualName (FirstName, MiddleName?, LastName, Suffix?, %affiliation; , Identifier*)> + +<!ELEMENT ELocationID (#PCDATA)> +<!ATTLIST ELocationID + EIdType (doi | pii) #REQUIRED + ValidYN (Y | N) "Y" + > +<!ELEMENT CopyrightInformation (#PCDATA)> +<!ELEMENT CoiStatement (%data;)* > +<!ELEMENT ArchiveCopySource (#PCDATA)> +<!ATTLIST ArchiveCopySource DocType CDATA #FIXED "pdf"> diff --git a/tests/sps/formats/test_pubmed.py b/tests/sps/formats/test_pubmed.py index a8afad598..08ea60da2 100644 --- a/tests/sps/formats/test_pubmed.py +++ b/tests/sps/formats/test_pubmed.py @@ -13,6 +13,7 @@ xml_pubmed_pub_date_pipe, xml_pubmed_article_title_pipe, xml_pubmed_first_page_pipe, + xml_pubmed_last_page_pipe, xml_pubmed_elocation_pipe, xml_pubmed_language_pipe, xml_pubmed_author_list, @@ -25,6 +26,12 @@ xml_pubmed_citations, xml_pubmed_abstract, xml_pubmed_other_abstract, + pipeline_pubmed, + pipeline_pubmed_set, + build_pubmed_article, + build_article_set_xml, + PUBMED_DOCTYPE, + MissingRequiredElementError, ) @@ -394,6 +401,48 @@ def test_xml_pubmed_pub_date_pipe(self): self.assertEqual(obtained, expected) + def test_xml_pubmed_pub_date_pipe_falls_back_to_collection_date(self): + # date-type="pub"/pub-type="epub" is absent; the only pub-date is + # pub-type="epub-ppub" (date-type="collection"), common in older + # SciELO articles -- article_date alone misses this, so get_date + # must fall back to collection_date instead of treating it as + # "no date at all". + expected = ( + '<Article>' + '<Journal>' + '<PubDate PubStatus="epublish">' + '<Year>2014</Year>' + '<Month>04</Month>' + '</PubDate>' + '</Journal>' + '</Article>' + ) + xml_pubmed = ET.fromstring( + '<Article>' + '<Journal/>' + '</Article>' + ) + xml_tree = ET.fromstring( + '<article xmlns:mml="http://www.w3.org/1998/Math/MathML" ' + 'xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'article-type="research-article" dtd-version="1.1" specific-use="sps-1.9" xml:lang="en">' + '<front>' + '<article-meta>' + '<pub-date pub-type="epub-ppub">' + '<month>04</month>' + '<year>2014</year>' + '</pub-date>' + '</article-meta>' + '</front>' + '</article>' + ) + + xml_pubmed_pub_date_pipe(xml_pubmed, xml_tree) + + obtained = ET.tostring(xml_pubmed, encoding="utf-8").decode("utf-8") + + self.assertEqual(obtained, expected) + def test_xml_pubmed_pub_date_pipe_without_day(self): expected = ( '<Article>' @@ -506,11 +555,6 @@ def test_xml_pubmed_pub_date_pipe_without_year(self): self.assertEqual(obtained, expected) def test_xml_pubmed_pub_date_pipe_without_date(self): - expected = ( - '<Article>' - '<Journal/>' - '</Article>' - ) xml_pubmed = ET.fromstring( '<Article>' '<Journal/>' @@ -527,11 +571,8 @@ def test_xml_pubmed_pub_date_pipe_without_date(self): '</article>' ) - xml_pubmed_pub_date_pipe(xml_pubmed, xml_tree) - - obtained = ET.tostring(xml_pubmed, encoding="utf-8").decode("utf-8") - - self.assertEqual(obtained, expected) + with self.assertRaises(MissingRequiredElementError): + xml_pubmed_pub_date_pipe(xml_pubmed, xml_tree) def test_xml_pubmed_article_title_pipe(self): expected = ( @@ -708,6 +749,89 @@ def test_xml_pubmed_first_page_pipe_without_first_page(self): self.assertEqual(obtained, expected) + def test_xml_pubmed_first_page_pipe_prefers_real_fpage_over_elocation_id(self): + # Articles with traditional pagination (//fpage) must not lose it -- + # elocation-id is only a fallback for continuous-publication articles + # that have no fpage at all. + expected = ( + '<Article>' + '<FirstPage LZero="save">227</FirstPage>' + '</Article>' + ) + xml_pubmed = ET.fromstring( + '<Article/>' + ) + xml_tree = ET.fromstring( + '<article xmlns:mml="http://www.w3.org/1998/Math/MathML" ' + 'xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'article-type="research-article" dtd-version="1.1" specific-use="sps-1.9" xml:lang="en">' + '<front>' + '<article-meta>' + '<fpage>227</fpage>' + '<elocation-id>e2022440</elocation-id>' + '</article-meta>' + '</front>' + '</article>' + ) + + xml_pubmed_first_page_pipe(xml_pubmed, xml_tree) + + obtained = ET.tostring(xml_pubmed, encoding="utf-8").decode("utf-8") + + self.assertEqual(obtained, expected) + + def test_xml_pubmed_last_page_pipe(self): + expected = ( + '<Article>' + '<LastPage>232</LastPage>' + '</Article>' + ) + xml_pubmed = ET.fromstring( + '<Article/>' + ) + xml_tree = ET.fromstring( + '<article xmlns:mml="http://www.w3.org/1998/Math/MathML" ' + 'xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'article-type="research-article" dtd-version="1.1" specific-use="sps-1.9" xml:lang="en">' + '<front>' + '<article-meta>' + '<fpage>227</fpage>' + '<lpage>232</lpage>' + '</article-meta>' + '</front>' + '</article>' + ) + + xml_pubmed_last_page_pipe(xml_pubmed, xml_tree) + + obtained = ET.tostring(xml_pubmed, encoding="utf-8").decode("utf-8") + + self.assertEqual(obtained, expected) + + def test_xml_pubmed_last_page_pipe_without_last_page(self): + expected = ( + '<Article/>' + ) + xml_pubmed = ET.fromstring( + '<Article/>' + ) + xml_tree = ET.fromstring( + '<article xmlns:mml="http://www.w3.org/1998/Math/MathML" ' + 'xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'article-type="research-article" dtd-version="1.1" specific-use="sps-1.9" xml:lang="en">' + '<front>' + '<article-meta>' + '</article-meta>' + '</front>' + '</article>' + ) + + xml_pubmed_last_page_pipe(xml_pubmed, xml_tree) + + obtained = ET.tostring(xml_pubmed, encoding="utf-8").decode("utf-8") + + self.assertEqual(obtained, expected) + def test_xml_pubmed_elocation_pipe(self): expected = ( '<Article>' @@ -1592,6 +1716,63 @@ def test_xml_pubmed_citations_without_article_list_id(self): self.assertEqual(obtained, expected) + def test_xml_pubmed_citations_creates_reference_list_when_ref_list_has_no_title(self): + # ref-list/title is optional in JATS. xml_pubmed_title_reference_list + # only creates <ReferenceList> when a title is present, so + # xml_pubmed_citations must create it itself when there are + # references but no title -- it used to crash with AttributeError + # ('NoneType' object has no attribute 'append') in this case. + expected = ( + '<Article>' + '<ReferenceList>' + '<Reference>' + '<Citation>Some citation text.</Citation>' + '</Reference>' + '</ReferenceList>' + '</Article>' + ) + xml_pubmed = ET.fromstring( + '<Article/>' + ) + xml_tree = ET.fromstring( + '<article xmlns:mml="http://www.w3.org/1998/Math/MathML" ' + 'xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'article-type="research-article" dtd-version="1.1" specific-use="sps-1.9" xml:lang="en">' + '<back>' + '<ref-list>' + '<ref id="B1">' + '<mixed-citation>Some citation text.</mixed-citation>' + '</ref>' + '</ref-list>' + '</back>' + '</article>' + ) + + xml_pubmed_citations(xml_pubmed, xml_tree) + + obtained = ET.tostring(xml_pubmed, encoding="utf-8").decode("utf-8") + + self.assertEqual(obtained, expected) + + def test_xml_pubmed_citations_without_refs_does_not_create_reference_list(self): + expected = '<Article/>' + xml_pubmed = ET.fromstring( + '<Article/>' + ) + xml_tree = ET.fromstring( + '<article xmlns:mml="http://www.w3.org/1998/Math/MathML" ' + 'xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'article-type="research-article" dtd-version="1.1" specific-use="sps-1.9" xml:lang="en">' + '<back></back>' + '</article>' + ) + + xml_pubmed_citations(xml_pubmed, xml_tree) + + obtained = ET.tostring(xml_pubmed, encoding="utf-8").decode("utf-8") + + self.assertEqual(obtained, expected) + def test_xml_pubmed_abstract_without_sections(self): self.maxDiff = None expected = ( @@ -1827,6 +2008,100 @@ def test_xml_pubmed_other_abstract(self): self.assertEqual(obtained, expected) + def test_pipeline_pubmed_wraps_article_in_article_set_with_doctype(self): + xml_tree = ET.fromstring( + '<article xmlns:mml="http://www.w3.org/1998/Math/MathML" ' + 'xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'article-type="research-article" dtd-version="1.1" specific-use="sps-1.9" xml:lang="en">' + '<front><article-meta>' + '<pub-date publication-format="electronic" date-type="pub">' + '<day>06</day><month>01</month><year>2023</year>' + '</pub-date>' + '</article-meta></front>' + '</article>' + ) + + obtained = pipeline_pubmed(xml_tree, pretty_print=False) + + self.assertTrue(obtained.startswith("<?xml version='1.0' encoding='utf-8'?>\n")) + self.assertIn(PUBMED_DOCTYPE, obtained) + self.assertIn('<ArticleSet><Article>', obtained) + self.assertTrue(obtained.rstrip().endswith('</Article></ArticleSet>')) + self.assertEqual(obtained.count('<Article>'), 1) + + def test_pipeline_pubmed_set_wraps_multiple_articles_in_single_article_set(self): + xml_tree = ET.fromstring( + '<article xmlns:mml="http://www.w3.org/1998/Math/MathML" ' + 'xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'article-type="research-article" dtd-version="1.1" specific-use="sps-1.9" xml:lang="en">' + '<front><article-meta>' + '<pub-date publication-format="electronic" date-type="pub">' + '<day>06</day><month>01</month><year>2023</year>' + '</pub-date>' + '</article-meta></front>' + '</article>' + ) + + obtained = pipeline_pubmed_set([xml_tree, xml_tree], pretty_print=False) + + self.assertIn(PUBMED_DOCTYPE, obtained) + self.assertEqual(obtained.count('<ArticleSet>'), 1) + self.assertEqual(obtained.count('<Article>'), 2) + self.assertEqual(obtained.count('</Article>'), 2) + + def test_pipeline_pubmed_set_raises_when_any_article_has_no_pub_date(self): + xml_tree_with_date = ET.fromstring( + '<article xmlns:mml="http://www.w3.org/1998/Math/MathML" ' + 'xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'article-type="research-article" dtd-version="1.1" specific-use="sps-1.9" xml:lang="en">' + '<front><article-meta>' + '<pub-date publication-format="electronic" date-type="pub">' + '<day>06</day><month>01</month><year>2023</year>' + '</pub-date>' + '</article-meta></front>' + '</article>' + ) + xml_tree_without_date = ET.fromstring( + '<article xmlns:mml="http://www.w3.org/1998/Math/MathML" ' + 'xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'article-type="research-article" dtd-version="1.1" specific-use="sps-1.9" xml:lang="en">' + '<front><article-meta></article-meta></front>' + '</article>' + ) + + with self.assertRaises(MissingRequiredElementError): + pipeline_pubmed_set([xml_tree_with_date, xml_tree_without_date]) + + def test_build_pubmed_article_raises_when_no_pub_date_is_found(self): + xml_tree = ET.fromstring( + '<article xmlns:mml="http://www.w3.org/1998/Math/MathML" ' + 'xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'article-type="research-article" dtd-version="1.1" specific-use="sps-1.9" xml:lang="en">' + '<front><article-meta></article-meta></front>' + '</article>' + ) + + with self.assertRaises(MissingRequiredElementError): + build_pubmed_article(xml_tree) + + def test_build_article_set_xml_wraps_pre_built_articles(self): + xml_tree = ET.fromstring( + '<article xmlns:mml="http://www.w3.org/1998/Math/MathML" ' + 'xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'article-type="research-article" dtd-version="1.1" specific-use="sps-1.9" xml:lang="en">' + '<front><article-meta>' + '<pub-date publication-format="electronic" date-type="pub">' + '<day>06</day><month>01</month><year>2023</year>' + '</pub-date>' + '</article-meta></front>' + '</article>' + ) + article = build_pubmed_article(xml_tree) + + obtained = build_article_set_xml([article], pretty_print=False) + + self.assertIn(PUBMED_DOCTYPE, obtained) + self.assertEqual(obtained.count('<Article>'), 1) if __name__ == '__main__': diff --git a/tests/sps/formats/test_pubmed_generator.py b/tests/sps/formats/test_pubmed_generator.py new file mode 100644 index 000000000..c6240cee5 --- /dev/null +++ b/tests/sps/formats/test_pubmed_generator.py @@ -0,0 +1,164 @@ +import os +import sys +import tempfile +import unittest +import zipfile + +from lxml import etree as ET + +from packtools.sps.formats.pubmed_generator import ( + build_articles_and_errors, + get_xml_trees_and_errors, + main, +) + +SAMPLE_1 = os.path.join( + os.path.dirname(__file__), "..", "..", "samples", "0034-7094-rba-69-03-0227.xml" +) +SAMPLE_2 = os.path.join( + os.path.dirname(__file__), "..", "..", "samples", "example.xml" +) + +XML_WITHOUT_PUB_DATE = ( + '<article xmlns:mml="http://www.w3.org/1998/Math/MathML" ' + 'xmlns:xlink="http://www.w3.org/1999/xlink" ' + 'article-type="research-article" dtd-version="1.1" specific-use="sps-1.9" xml:lang="en">' + '<front><article-meta></article-meta></front>' + '</article>' +) + + +class GetXmlTreesAndErrors(unittest.TestCase): + def test_single_xml_file(self): + xml_trees, errors = get_xml_trees_and_errors(SAMPLE_1) + + self.assertEqual(len(xml_trees), 1) + self.assertEqual(errors, []) + self.assertEqual(xml_trees[0].tag, "article") + + def test_zip_package_with_multiple_articles(self): + with tempfile.TemporaryDirectory() as tmp_dir: + zip_path = os.path.join(tmp_dir, "package.zip") + with zipfile.ZipFile(zip_path, "w") as zf: + zf.write(SAMPLE_1, arcname="article1.xml") + zf.write(SAMPLE_2, arcname="article2.xml") + + xml_trees, errors = get_xml_trees_and_errors(zip_path) + + self.assertEqual(len(xml_trees), 2) + self.assertEqual(errors, []) + self.assertTrue(all(xml_tree.tag == "article" for xml_tree in xml_trees)) + + def test_zip_package_skips_invalid_xml_and_reports_error(self): + with tempfile.TemporaryDirectory() as tmp_dir: + zip_path = os.path.join(tmp_dir, "package.zip") + with zipfile.ZipFile(zip_path, "w") as zf: + zf.write(SAMPLE_1, arcname="valid.xml") + zf.writestr("invalid.xml", "this is not xml") + + xml_trees, errors = get_xml_trees_and_errors(zip_path) + + self.assertEqual(len(xml_trees), 1) + self.assertEqual(len(errors), 1) + self.assertEqual(errors[0].get("filename"), "invalid.xml") + self.assertIsNotNone(errors[0].get("error")) + + +class BuildArticlesAndErrors(unittest.TestCase): + def test_builds_one_article_per_valid_xml_tree(self): + xml_trees, _ = get_xml_trees_and_errors(SAMPLE_1) + + articles, errors = build_articles_and_errors(xml_trees) + + self.assertEqual(len(articles), 1) + self.assertEqual(errors, []) + + def test_skips_article_without_pub_date_and_reports_error(self): + xml_trees, _ = get_xml_trees_and_errors(SAMPLE_1) + xml_trees.append(ET.fromstring(XML_WITHOUT_PUB_DATE)) + + articles, errors = build_articles_and_errors(xml_trees) + + self.assertEqual(len(articles), 1) + self.assertEqual(len(errors), 1) + self.assertIsNotNone(errors[0].get("error")) + + +class Main(unittest.TestCase): + def test_main_writes_article_set_from_single_xml_file(self): + with tempfile.TemporaryDirectory() as tmp_dir: + output_path = os.path.join(tmp_dir, "out.xml") + argv = sys.argv + try: + sys.argv = [ + "pubmed_generator", + "-i", SAMPLE_1, + "-o", output_path, + ] + main() + finally: + sys.argv = argv + + with open(output_path, encoding="utf-8") as fp: + content = fp.read() + + self.assertIn("<!DOCTYPE ArticleSet", content) + self.assertEqual(content.count("<Article>"), 1) + # garante que o XML gerado é bem formado + ET.fromstring(content.encode("utf-8")) + + def test_main_writes_article_set_from_zip_package(self): + with tempfile.TemporaryDirectory() as tmp_dir: + zip_path = os.path.join(tmp_dir, "package.zip") + with zipfile.ZipFile(zip_path, "w") as zf: + zf.write(SAMPLE_1, arcname="article1.xml") + zf.write(SAMPLE_2, arcname="article2.xml") + + output_path = os.path.join(tmp_dir, "out.xml") + argv = sys.argv + try: + sys.argv = [ + "pubmed_generator", + "-i", zip_path, + "-o", output_path, + ] + main() + finally: + sys.argv = argv + + with open(output_path, encoding="utf-8") as fp: + content = fp.read() + + self.assertIn("<!DOCTYPE ArticleSet", content) + self.assertEqual(content.count("<Article>"), 2) + ET.fromstring(content.encode("utf-8")) + + def test_main_skips_article_without_pub_date_and_still_writes_the_rest(self): + with tempfile.TemporaryDirectory() as tmp_dir: + zip_path = os.path.join(tmp_dir, "package.zip") + with zipfile.ZipFile(zip_path, "w") as zf: + zf.write(SAMPLE_1, arcname="article1.xml") + zf.writestr("no_pub_date.xml", XML_WITHOUT_PUB_DATE) + + output_path = os.path.join(tmp_dir, "out.xml") + argv = sys.argv + try: + sys.argv = [ + "pubmed_generator", + "-i", zip_path, + "-o", output_path, + ] + main() + finally: + sys.argv = argv + + with open(output_path, encoding="utf-8") as fp: + content = fp.read() + + self.assertIn("<!DOCTYPE ArticleSet", content) + self.assertEqual(content.count("<Article>"), 1) + ET.fromstring(content.encode("utf-8")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/sps/formats/test_pubmed_integration.py b/tests/sps/formats/test_pubmed_integration.py new file mode 100644 index 000000000..9e2c70f43 --- /dev/null +++ b/tests/sps/formats/test_pubmed_integration.py @@ -0,0 +1,91 @@ +import glob +import os +import unittest + +from lxml import etree as ET + +from packtools.sps.formats import pubmed + +HERE = os.path.dirname(os.path.abspath(__file__)) +DTD_PATH = os.path.join(HERE, "..", "..", "fixtures", "pubmed", "PubMed.dtd") +SAMPLES_DIR = os.path.join(HERE, "..", "..", "samples") + +# Samples that don't produce DTD-valid PubMed XML yet on this branch because +# the fix lives in an independent, not-yet-merged sub-issue PR of #1226. +# When one of these unexpectedly starts passing, `assertRaises` below will +# fail loudly -- that's the signal to remove the corresponding entry. +KNOWN_FAILURES = { + "example.xml": ( + "Author sem <surname> (só <given-names>) produz Author sem " + "LastName -- corrigido em #1236/PR #1245, ainda não mesclado aqui." + ), + "0034-8910-rsp-48-2-0249.xml": ( + "contrib sem xref de afiliação faz get_affiliations quebrar com " + "TypeError -- corrigido com `or []` em #1236/PR #1245, ainda não " + "mesclado aqui." + ), +} + + +def _sample_paths(): + return sorted(glob.glob(os.path.join(SAMPLES_DIR, "*.xml"))) + + +def _build_article_set_xml(path): + xml_tree = ET.parse(path).getroot() + article = pubmed.build_pubmed_article(xml_tree) + return pubmed.build_article_set_xml([article]) + + +class PubmedRealSamplesEndToEndTest(unittest.TestCase): + """ + Testes de integração fim-a-fim: cada amostra real em tests/samples/ + passa pelo pipeline completo (build_pubmed_article + build_article_set_xml) + e o resultado é validado contra a DTD real do PubMed (vendorizada em + tests/fixtures/pubmed/PubMed.dtd, para não depender de rede). Diferente + de test_pubmed.py, que testa cada pipe isoladamente com XML sintético + mínimo, aqui o objetivo é pegar regressões que só aparecem com artigos + reais e completos (datas, paginação, referências, etc. combinados). + """ + + @classmethod + def setUpClass(cls): + cls.dtd = ET.DTD(DTD_PATH) + + def _assert_produces_valid_pubmed_xml(self, path): + xml_set = _build_article_set_xml(path) + doc = ET.fromstring(xml_set.encode("utf-8")) + valid = self.dtd.validate(doc) + if not valid: + self.fail(str(self.dtd.error_log.filter_from_errors())) + + def test_real_samples_produce_dtd_valid_pubmed_xml(self): + paths = _sample_paths() + self.assertTrue(paths, "nenhuma amostra encontrada em tests/samples/*.xml") + + for path in paths: + name = os.path.basename(path) + with self.subTest(sample=name): + if name in KNOWN_FAILURES: + with self.assertRaises( + (AssertionError, TypeError), + msg=( + f"{name} passou a gerar XML válido -- remova a " + f"entrada em KNOWN_FAILURES ({KNOWN_FAILURES[name]})" + ), + ): + self._assert_produces_valid_pubmed_xml(path) + else: + self._assert_produces_valid_pubmed_xml(path) + + def test_no_stale_known_failures(self): + """Garante que KNOWN_FAILURES só referencia amostras que existem.""" + existing = {os.path.basename(p) for p in _sample_paths()} + stale = set(KNOWN_FAILURES) - existing + self.assertFalse( + stale, f"KNOWN_FAILURES referencia amostras inexistentes: {stale}" + ) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file