bioconverters

362def parse_pmcxml(
363    source: Union[str, TextIO],
364    return_xml: bool = False,
365    keep_tags=set(),
366    trim_buggy_sentences: bool = True,
367    inject_citations: bool = False,
368    clean_numeric_citations: bool = True,
369    clean_xrefs_in_brackets: bool = True,
370    clear_empty_brackets: bool = True,
371    fix_exponentials: bool = True,
372    strip_tag_attributes: bool = True,
373) -> Iterable[PMCArticle]:
374    """
375    Parse a PMC XML file into a series of PMCArticle objects (one per article/sub-article).
376
377    Args:
378        source: The text or file handle containing the PMC XML
379        return_xml: return each passage's text as a marked-up XML string if True, or as
380            plain, unescaped text with any markup stripped if False (default).
381        keep_tags: with return_xml=True, tags whose markup is preserved inline in each
382            passage's text (e.g. "sup", "italic") - has no effect when return_xml=False.
383            Defaults to an empty set (no markup kept). Pass `pmc_constants.PMC_KEEP_TAGS`
384            for the common formatting tags (sup, sub, italic, bold, underline, monospace,
385            sc, overline, strike).
386        trim_buggy_sentences: trim overly long, unbroken runs of text to a maximum length,
387            to avoid issues with buggy sentences in some PMC articles.
388        inject_citations: resolve each in-text bibr citation's pmid/doi and retag it to
389            `<citation>`, kept in the output instead of dropped. Must not be combined with
390            clean_numeric_citations=True - injection enriches bibr citations,
391            clean_numeric_citations deletes them, so having both on is almost certainly a mistake.
392        clean_numeric_citations: drop a numeric bibr citation marker outright, e.g. "1", "[1,2]",
393            regardless of context - this is what catches a citation glued directly onto a
394            word with no separating space at all.
395        clean_xrefs_in_brackets: drop bracket-wrapped xref clutter, e.g. "(Table 1)".
396        clear_empty_brackets: remove any "(...)"/"[...]"/"{...}" left containing no word
397            characters, e.g. from clean_numeric_citations/clean_xrefs_in_brackets, or from an
398            unrelated ignore_tag (like ext-link) that happened to be parenthesised.
399        fix_exponentials: with return_xml=False, recover a digit-preceded numeric `<sup>`
400            as "^N" instead of losing it to plain concatenation, e.g. `"10<sup>8</sup>"` ->
401            "10^8".
402        strip_tag_attributes: with return_xml=True, strip source-XML attributes (e.g. the
403            "toggle" in `<italic toggle="yes">`) from every kept tag, since they're clutter
404            with no meaning to callers - has no effect when return_xml=False, since markup is
405            stripped entirely then. Citation injection's own attributes (pmid/doi/count) are
406            always kept regardless of this flag, since they carry meaning that plain markup
407            can't express.
408
409    These defaults match pmcxml2bioc/pmcxml2txt, so behavior is consistent regardless of
410    which entry point is used.
411    """
412    assert not (inject_citations and clean_numeric_citations), (
413        "inject_citations and clean_numeric_citations can't both be True - injection enriches bibr "
414        "citations, clean_numeric_citations deletes them. Pass clean_numeric_citations=False."
415    )
416
417    source = _apply_pmc_xlink_fix(source)
418
419    # Skip to the article element in the file
420    for event, elem in etree.iterparse(source, events=("start", "end", "start-ns", "end-ns")):
421        if event == "end" and elem.tag == "article":
422            if inject_citations:
423                citation_lookup = _build_citation_lookup(elem)
424                _inject_citations(elem, citation_lookup)
425
426            meta = _get_meta_info_for_pmc_article(elem)
427
428            # We're going to process the main article along with any subarticles
429            # And if any of the subarticles have distinguishing IDs (e.g. PMID), then
430            # that'll be used, otherwise the parent article's metadata will be used
431            subarticles = [elem] + elem.findall("./sub-article")
432
433            for article_elem in subarticles:
434                if article_elem is elem:
435                    # This is the main parent article. Just use its metadata
436                    sub_meta = meta
437                else:
438                    # Check if this subarticle has any distinguishing IDs and use them instead
439                    sub_meta = _get_meta_info_for_pmc_article(article_elem)
440                    if not (sub_meta.pmid or sub_meta.pmcid or sub_meta.doi):
441                        sub_meta.pmid = meta.pmid
442                        sub_meta.pmcid = meta.pmcid
443                        sub_meta.doi = meta.doi
444                    if sub_meta.pub_year is None:
445                        sub_meta.pub_year = meta.pub_year
446                        sub_meta.pub_month = meta.pub_month
447                        sub_meta.pub_day = meta.pub_day
448                    if not sub_meta.journal:
449                        sub_meta.journal = meta.journal
450                        sub_meta.journal_iso = meta.journal_iso
451
452                content = _extract_article_content(
453                    article_elem,
454                    keep_tags,
455                    return_xml,
456                    trim_buggy_sentences,
457                    inject_citations,
458                    clean_numeric_citations,
459                    clean_xrefs_in_brackets,
460                    clear_empty_brackets,
461                    fix_exponentials,
462                    strip_tag_attributes,
463                )
464
465                yield PMCArticle(
466                    pmcid=sub_meta.pmcid,
467                    pmid=sub_meta.pmid,
468                    doi=sub_meta.doi,
469                    pub_year=sub_meta.pub_year,
470                    pub_month=sub_meta.pub_month,
471                    pub_day=sub_meta.pub_day,
472                    journal=sub_meta.journal,
473                    journal_iso=sub_meta.journal_iso,
474                    **content,
475                )
476
477            # Less important here (compared to abstracts) as each article file is not too big
478            elem.clear()

481def pmcxml2bioc(
482    source: Union[str, TextIO],
483    sections: Iterable[str] = ("title", "subtitle", "abstract", "article", "back", "floating"),
484    trim_buggy_sentences: bool = True,
485    clean_numeric_citations: bool = True,
486    clean_xrefs_in_brackets: bool = True,
487    clear_empty_brackets: bool = True,
488    fix_exponentials: bool = True,
489) -> Iterator[bioc.BioCDocument]:
490    """
491    Convert a PMC XML file into its Bioc equivalent
492
493    Args:
494        source: The text or file handle containing the PMC XML
495        sections: which of the six PMCArticle text fields ("title", "subtitle", "abstract",
496            "article", "back", "floating") to include as passages, and in what order.
497        trim_buggy_sentences: trim overly long, unbroken runs of text to a maximum length,
498            to avoid issues with buggy sentences in some PMC articles.
499        clean_numeric_citations: see parse_pmcxml.
500        clean_xrefs_in_brackets: see parse_pmcxml.
501        clear_empty_brackets: see parse_pmcxml.
502        fix_exponentials: see parse_pmcxml.
503
504    Raises:
505        RuntimeError: On any parsing errors
506
507    Returns:
508        An iterator over the newly generated Bioc documents
509    """
510    try:
511        for pmc_doc in parse_pmcxml(
512            source,
513            keep_tags=set(),
514            return_xml=False,
515            inject_citations=False,
516            trim_buggy_sentences=trim_buggy_sentences,
517            clean_numeric_citations=clean_numeric_citations,
518            clean_xrefs_in_brackets=clean_xrefs_in_brackets,
519            clear_empty_brackets=clear_empty_brackets,
520            fix_exponentials=fix_exponentials,
521        ):
522            bioc_doc = bioc.BioCDocument()
523            bioc_doc.id = pmc_doc.pmid
524            bioc_doc.infons["title"] = pmc_doc.title
525            bioc_doc.infons["pmid"] = pmc_doc.pmid
526            bioc_doc.infons["pmcid"] = pmc_doc.pmcid
527            bioc_doc.infons["doi"] = pmc_doc.doi
528            bioc_doc.infons["year"] = pmc_doc.pub_year
529            bioc_doc.infons["month"] = pmc_doc.pub_month
530            bioc_doc.infons["day"] = pmc_doc.pub_day
531            bioc_doc.infons["journal"] = pmc_doc.journal
532            bioc_doc.infons["journal_iso"] = pmc_doc.journal_iso
533
534            offset = 0
535            for group_name in sections:
536                value = getattr(pmc_doc, group_name)
537                texts = (value,) if isinstance(value, str) else value
538                for text_source in texts:
539                    if not text_source:
540                        continue
541
542                    passage = bioc.BioCPassage()
543
544                    passage.infons["section"] = group_name
545
546                    passage.text = text_source
547                    passage.offset = offset
548
549                    offset += len(text_source)
550                    bioc_doc.add_passage(passage)
551
552            yield bioc_doc
553
554    except etree.ParseError:
555        raise RuntimeError("Parsing error in PMC xml file: %s" % source)

572def pmcxml2txt(
573    source: Union[str, TextIO],
574    sections: Iterable[str] = ("title", "subtitle", "abstract", "article", "back", "floating"),
575    passage_separator: str = "\n\n",
576    trim_buggy_sentences: bool = True,
577    clean_numeric_citations: bool = True,
578    clean_xrefs_in_brackets: bool = True,
579    clear_empty_brackets: bool = True,
580    fix_exponentials: bool = True,
581) -> Iterator[Tuple[PMCMeta, str]]:
582    """
583    Convert a PMC XML file into plain text, one (metadata, text) pair per article/sub-article.
584
585    Args:
586        source: The text or file handle containing the PMC XML
587        sections: which of the six PMCArticle text fields ("title", "subtitle", "abstract",
588            "article", "back", "floating") to include, and in what order.
589        passage_separator: string used to join the extracted passages into the single
590            returned text string.
591        trim_buggy_sentences: trim overly long, unbroken runs of text to a maximum length,
592            to avoid issues with buggy sentences in some PMC articles.
593        clean_numeric_citations: see parse_pmcxml.
594        clean_xrefs_in_brackets: see parse_pmcxml.
595        clear_empty_brackets: see parse_pmcxml.
596        fix_exponentials: see parse_pmcxml.
597
598    Returns:
599        An iterator over one (PMCMeta, text) pair per article/sub-article
600    """
601    for doc in parse_pmcxml(
602        source,
603        keep_tags=set(),
604        return_xml=False,
605        trim_buggy_sentences=trim_buggy_sentences,
606        clean_numeric_citations=clean_numeric_citations,
607        clean_xrefs_in_brackets=clean_xrefs_in_brackets,
608        clear_empty_brackets=clear_empty_brackets,
609        fix_exponentials=fix_exponentials,
610    ):
611        text = passage_separator.join(doc.iter_text(sections))
612        yield _pmc_article_meta(doc), text

615def pmcxml2tagged(
616    source: Union[str, TextIO],
617    sections: Iterable[str] = ("title", "subtitle", "abstract", "article", "back", "floating"),
618    passage_separator: str = "\n\n",
619    keep_tags=PMC_KEEP_TAGS,
620    trim_buggy_sentences: bool = True,
621    strip_tag_attributes: bool = True,
622) -> Iterator[Tuple[PMCMeta, str]]:
623    """
624    Convert a PMC XML file into marked-up text, one (metadata, text) pair per
625    article/sub-article, with formatting tags (e.g. `<sup>`, `<italic>`) kept inline and
626    in-text citations resolved and kept as `<citation pmid="...">` instead of stripped. A
627    thin wrapper around parse_pmcxml with return_xml=True, keep_tags defaulted to
628    PMC_KEEP_TAGS, and inject_citations=True (so clean_numeric_citations, which can't be
629    combined with it, is forced off). clean_xrefs_in_brackets/clear_empty_brackets/
630    fix_exponentials are forced off too - this is meant to keep markup/citations intact
631    rather than clean up the text, and fix_exponentials has no effect under return_xml=True
632    anyway (see parse_pmcxml).
633
634    Args:
635        source: The text or file handle containing the PMC XML
636        sections: which of the six PMCArticle text fields ("title", "subtitle", "abstract",
637            "article", "back", "floating") to include, and in what order.
638        passage_separator: string used to join the extracted passages into the single
639            returned text string.
640        keep_tags: see parse_pmcxml. Defaults to `pmc_constants.PMC_KEEP_TAGS`.
641        trim_buggy_sentences: trim overly long, unbroken runs of text to a maximum length,
642            to avoid issues with buggy sentences in some PMC articles.
643        strip_tag_attributes: see parse_pmcxml.
644
645    Returns:
646        An iterator over one (PMCMeta, marked-up text) pair per article/sub-article
647    """
648    for doc in parse_pmcxml(
649        source,
650        keep_tags=keep_tags,
651        return_xml=True,
652        trim_buggy_sentences=trim_buggy_sentences,
653        inject_citations=True,
654        clean_numeric_citations=False,
655        clean_xrefs_in_brackets=False,
656        clear_empty_brackets=False,
657        fix_exponentials=False,
658        strip_tag_attributes=strip_tag_attributes,
659    ):
660        text = passage_separator.join(doc.iter_text(sections))
661        yield _pmc_article_meta(doc), text

 38@dataclass
 39class PMCArticle(PMCMeta):
 40    """One PMC article or sub-article, as extracted by `parse_pmcxml`."""
 41
 42    # Redeclared here (not just inherited from PMCMeta) purely so pdoc documents them on
 43    # this class - it doesn't inline a base class's fields into a subclass's page. Dataclass
 44    # field order/behavior is unaffected: re-annotating an inherited field updates its type
 45    # in place without moving it, since ordering is fixed by each name's first occurrence
 46    # when walking the MRO.
 47    pmcid: str
 48    """PubMed Central ID (this article's own primary key), or an empty string if not found."""
 49
 50    pmid: Optional[str]
 51    """PubMed ID, or None if this article isn't linked to one (e.g. not PubMed-indexed)."""
 52
 53    doi: Optional[str]
 54    """DOI, or None if this article doesn't have one registered."""
 55
 56    pub_year: Optional[str]
 57    """Publication year, or None if not found."""
 58
 59    pub_month: Optional[Union[str, int]]
 60    """Publication month, or None if not found."""
 61
 62    pub_day: Optional[str]
 63    """Publication day, or None if not found."""
 64
 65    journal: str
 66    """Journal title, or an empty string if not found."""
 67
 68    journal_iso: str
 69    """ISO abbreviation of the journal title, or an empty string if not found."""
 70
 71    title: str
 72    """Article title, or an empty string if not found."""
 73
 74    subtitle: str
 75    """Article subtitle, or an empty string if not present."""
 76
 77    abstract: Iterable[str]
 78    """Abstract passages."""
 79
 80    article: Iterable[str]
 81    """Body text passages, extracted from the article's `<body>` element."""
 82
 83    back: Iterable[str]
 84    """Back-matter text passages (e.g. appendices), extracted from the article's `<back>` element."""
 85
 86    floating: Iterable[str]
 87    """Text passages from blocks outside the main flow (e.g. figure/table captions),
 88    extracted from the article's `<floats-group>` element."""
 89
 90    def iter_text(self, sections: Iterable[str] = _ALL_SECTIONS) -> Iterator[str]:
 91        """
 92        Yield each non-empty passage of text from the given fields, in order.
 93
 94        Args:
 95            sections: which of the six text fields to pull text from, and in what order.
 96                Defaults to all six ("title", "subtitle", "abstract", "article", "back",
 97                "floating").
 98        """
 99        for section in sections:
100            value = getattr(self, section)
101            texts = (value,) if isinstance(value, str) else value
102            for text in texts:
103                if text:
104                    yield text

 8@dataclass
 9class PMCMeta:
10    """Metadata common to all PMC articles and sub-articles - the text-free subset of
11    `PMCArticle`'s fields, as returned alongside the text by `pmcxml2txt`/`pmcxml2tagged`."""
12    
13    pmcid: str
14    """PubMed Central ID (this article's own primary key), or an empty string if not found."""
15
16    pmid: Optional[str]
17    """PubMed ID, or None if this article isn't linked to one (e.g. not PubMed-indexed)."""
18
19    doi: Optional[str]
20    """DOI, or None if this article doesn't have one registered."""
21
22    pub_year: Optional[str]
23    """Publication year, or None if not found."""
24
25    pub_month: Optional[Union[str, int]]
26    """Publication month, or None if not found."""
27
28    pub_day: Optional[str]
29    """Publication day, or None if not found."""
30
31    journal: str
32    """Journal title, or an empty string if not found."""
33
34    journal_iso: str
35    """ISO abbreviation of the journal title, or an empty string if not found."""

135def parse_pubmedxml(
136    source: Union[str, TextIO],
137    return_xml: bool = False,
138    keep_tags=set(),
139    clear_empty_brackets: bool = True,
140    fix_exponentials: bool = True,
141) -> Iterable[PubMedArticle]:
142    """
143    Args:
144        source: path to the MEDLINE xml file
145        return_xml: return the title/abstract text as a marked-up XML string if True, or as
146            plain, unescaped text with any markup stripped if False (default).
147        keep_tags: with return_xml=True, tags whose markup is preserved inline (e.g. "i",
148            "b", "sup") - has no effect when return_xml=False. Defaults to an empty set (no
149            markup kept). Pass `pubmed_constants.PUBMED_KEEP_TAGS` for the common formatting
150            tags (i, b, u, sup, sub).
151        clear_empty_brackets: remove any "(...)"/"[...]"/"{...}" left containing no word
152            characters.
153        fix_exponentials: recover a digit-preceded numeric `<sup>` as "^N" instead of losing
154            it to plain concatenation, e.g. `"10<sup>8</sup>"` -> "10^8". Same default as
155            pubmedxml2txt/pubmedxml2bioc. Requires "sup" to be preserved, which happens
156            automatically regardless of keep_tags.
157    """
158    effective_keep_tags = keep_tags | {"sup"} if fix_exponentials else keep_tags
159    for event, elem in etree.iterparse(source, events=("start", "end", "start-ns", "end-ns")):
160        if event == "end" and elem.tag == "PubmedArticle":  # MedlineCitation'):
161            # Try to extract the pmid_id
162            pmid_field = elem.find("./MedlineCitation/PMID")
163            assert pmid_field is not None
164            pmid = pmid_field.text
165
166            journal_year, journal_month, journal_day = _get_journal_date_for_medline_file(elem, pmid)
167            entry_year, entry_month, entry_day = _get_pubmed_entry_date(elem)
168
169            j_comparison = tuple(
170                9999 if d is None else d for d in [journal_year, journal_month, journal_day]
171            )
172            e_comparison = tuple(
173                9999 if d is None else d for d in [entry_year, entry_month, entry_day]
174            )
175            if (
176                j_comparison < e_comparison
177            ):  # The PubMed entry has been delayed for some reason so let's try the journal data
178                pub_year, pub_month, pub_day = journal_year, journal_month, journal_day
179            else:
180                pub_year, pub_month, pub_day = entry_year, entry_month, entry_day
181
182            # Extract the authors
183            author_elems = elem.findall("./MedlineCitation/Article/AuthorList/Author")
184            authors = []
185            for author_elem in author_elems:
186                forename = author_elem.find("./ForeName")
187                lastname = author_elem.find("./LastName")
188                collectivename = author_elem.find("./CollectiveName")
189
190                name = None
191                if (
192                    forename is not None
193                    and lastname is not None
194                    and forename.text is not None
195                    and lastname.text is not None
196                ):
197                    name = "%s %s" % (forename.text, lastname.text)
198                elif lastname is not None and lastname.text is not None:
199                    name = lastname.text
200                elif forename is not None and forename.text is not None:
201                    name = forename.text
202                elif collectivename is not None and collectivename.text is not None:
203                    name = collectivename.text
204                else:
205                    raise RuntimeError("Unable to find authors in Pubmed citation (PMID=%s)" % pmid)
206                authors.append(name)
207
208            chemicals = []
209            chemical_elems = elem.findall("./MedlineCitation/ChemicalList/Chemical")
210            for chemical_elem in chemical_elems:
211                substance_elem = chemical_elem.find("./NameOfSubstance")
212                chemicals.append(
213                    Chemical(
214                        ui=substance_elem.attrib["UI"],
215                        name=substance_elem.text,
216                        registry_number=chemical_elem.find("./RegistryNumber").text,
217                    )
218                )
219
220            mesh_headings = []
221            mesh_elems = elem.findall("./MedlineCitation/MeshHeadingList/MeshHeading")
222            for mesh_elem in mesh_elems:
223                descriptor_elem = mesh_elem.find("./DescriptorName")
224                qualifiers = [
225                    MeshQualifier(
226                        ui=qualifier_elem.attrib["UI"],
227                        name=qualifier_elem.text,
228                        major_topic=qualifier_elem.attrib["MajorTopicYN"] == "Y",
229                    )
230                    for qualifier_elem in mesh_elem.findall("./QualifierName")
231                ]
232                mesh_headings.append(
233                    MeshHeading(
234                        ui=descriptor_elem.attrib["UI"],
235                        name=descriptor_elem.text,
236                        major_topic=descriptor_elem.attrib["MajorTopicYN"] == "Y",
237                        qualifiers=qualifiers,
238                    )
239                )
240
241            supplementary_concepts = [
242                SupplementaryMeshConcept(
243                    ui=concept_elem.attrib["UI"],
244                    type=concept_elem.attrib["Type"],
245                    name=concept_elem.text,
246                )
247                for concept_elem in elem.findall("./MedlineCitation/SupplMeshList/SupplMeshName")
248            ]
249
250            doi_elems = elem.findall("./PubmedData/ArticleIdList/ArticleId[@IdType='doi']")
251            dois = [
252                doi_elem.text
253                for doi_elem in doi_elems
254                if doi_elem.text and _doi_regex.match(doi_elem.text)
255            ]
256
257            doi = None
258            if dois:
259                doi = dois[0]  # We'll just use DOI the first one provided
260
261            pmc_elems = elem.findall("./PubmedData/ArticleIdList/ArticleId[@IdType='pmc']")
262            assert len(pmc_elems) <= 1, "Found more than one PMCID with PMID: %s" % pmid
263            pmcid = None
264            if len(pmc_elems) == 1:
265                pmcid = pmc_elems[0].text
266
267            pub_type_elems = elem.findall(
268                "./MedlineCitation/Article/PublicationTypeList/PublicationType"
269            )
270            publication_types = [
271                PublicationType(ui=e.attrib["UI"], name=e.text)
272                for e in pub_type_elems
273                if e.text not in _pub_type_skips
274            ]
275
276            # Extract the title of paper - the DTD requires exactly one ArticleTitle per Article
277            title = elem.findall("./MedlineCitation/Article/ArticleTitle")
278            assert len(title) == 1, "Expected exactly one ArticleTitle for PMID=%s" % pmid
279            title_passages = _extract_passages(
280                title,
281                PUBMED_IGNORE_TAGS,
282                PUBMED_SPLIT_TAGS,
283                effective_keep_tags,
284                return_xml=return_xml,
285                trim_buggy_sentences=True,
286                fix_exponentials=fix_exponentials,
287            )
288            title_text = _remove_brackets_from_titles(title_passages[0]) if title_passages else ""
289            if not return_xml:
290                # html.unescape catches named entities (e.g. &alpha;) that XML unescaping
291                # alone doesn't - only safe on plain text, since unescaping "&amp;" etc. back
292                # to "&" inside return_xml=True's own markup would corrupt it
293                title_text = html.unescape(title_text)
294            if clear_empty_brackets:
295                title_text = _remove_brackets_without_words(title_text)
296
297            # Extract the abstract from the paper
298            abstract = elem.findall("./MedlineCitation/Article/Abstract/AbstractText")
299            abstract_passages = _extract_passages(
300                abstract,
301                PUBMED_IGNORE_TAGS,
302                PUBMED_SPLIT_TAGS,
303                effective_keep_tags,
304                return_xml=return_xml,
305                trim_buggy_sentences=True,
306                fix_exponentials=fix_exponentials,
307            )
308            abstract_text = (
309                abstract_passages if return_xml else [html.unescape(t) for t in abstract_passages]
310            )
311            if clear_empty_brackets:
312                abstract_text = [_remove_brackets_without_words(t) for t in abstract_text]
313
314            journal_title_fields = elem.findall("./MedlineCitation/Article/Journal/Title")
315            journal_title_iso_fields = elem.findall(
316                "./MedlineCitation/Article/Journal/ISOAbbreviation"
317            )
318
319            journal_title, journal_iso_title = "", ""
320            assert len(journal_title_fields) <= 1, "Error with pmid=%s" % pmid
321            assert len(journal_title_iso_fields) <= 1, "Error with pmid=%s" % pmid
322            if journal_title_fields:
323                journal_title = journal_title_fields[0].text
324            if journal_title_iso_fields:
325                journal_iso_title = journal_title_iso_fields[0].text
326
327            yield PubMedArticle(
328                pmid=pmid,
329                pmcid=pmcid,
330                doi=doi,
331                pub_year=pub_year,
332                pub_month=pub_month,
333                pub_day=pub_day,
334                title=title_text,
335                abstract=abstract_text,
336                journal=journal_title,
337                journal_iso=journal_iso_title,
338                authors=authors,
339                chemicals=chemicals,
340                mesh_headings=mesh_headings,
341                supplementary_mesh=supplementary_concepts,
342                publication_types=publication_types,
343            )
344
345            # Important: clear the current element from memory to keep memory usage low
346            elem.clear()

369def pubmedxml2bioc(
370    source: Union[str, TextIO],
371    sections: Iterable[str] = ("title", "abstract"),
372    clear_empty_brackets: bool = True,
373    fix_exponentials: bool = True,
374) -> Iterable[bioc.BioCDocument]:
375    """
376    Args:
377        source: path to the MEDLINE xml file
378        sections: which of "title"/"abstract" to include, and in what order.
379        clear_empty_brackets: see parse_pubmedxml.
380        fix_exponentials: see parse_pubmedxml.
381    """
382    for pm_doc in parse_pubmedxml(
383        source, clear_empty_brackets=clear_empty_brackets, fix_exponentials=fix_exponentials
384    ):
385        bioc_doc = bioc.BioCDocument()
386        bioc_doc.id = pm_doc.pmid
387        bioc_doc.infons["title"] = pm_doc.title
388        bioc_doc.infons["pmid"] = pm_doc.pmid
389        bioc_doc.infons["pmcid"] = pm_doc.pmcid
390        bioc_doc.infons["doi"] = pm_doc.doi
391        bioc_doc.infons["year"] = pm_doc.pub_year
392        bioc_doc.infons["month"] = pm_doc.pub_month
393        bioc_doc.infons["day"] = pm_doc.pub_day
394        bioc_doc.infons["journal"] = pm_doc.journal
395        bioc_doc.infons["journal_iso"] = pm_doc.journal_iso
396        bioc_doc.infons["authors"] = ", ".join(pm_doc.authors)
397        bioc_doc.infons["chemicals"] = "\t".join(_format_chemical_for_infons(c) for c in pm_doc.chemicals)
398        bioc_doc.infons["mesh_headings"] = "\t".join(
399            _format_mesh_heading_for_infons(h) for h in pm_doc.mesh_headings
400        )
401        bioc_doc.infons["supplementary_mesh"] = "\t".join(
402            _format_supplementary_mesh_for_infons(s) for s in pm_doc.supplementary_mesh
403        )
404        bioc_doc.infons["publication_types"] = "\t".join(
405            _format_publication_type_for_infons(p) for p in pm_doc.publication_types
406        )
407
408        offset = 0
409        for section in sections:
410            value = getattr(pm_doc, section)
411            texts = (value,) if isinstance(value, str) else value
412            for text_source in texts:
413                if not text_source:
414                    continue
415                passage = bioc.BioCPassage()
416                passage.infons["section"] = section
417                passage.text = text_source
418                passage.offset = offset
419                offset += len(text_source)
420                bioc_doc.add_passage(passage)
421
422        yield bioc_doc

444def pubmedxml2txt(
445    source: Union[str, TextIO],
446    sections: Iterable[str] = ("title", "abstract"),
447    passage_separator: str = "\n\n",
448    clear_empty_brackets: bool = True,
449    fix_exponentials: bool = True,
450) -> Iterator[Tuple[PubMedMeta, str]]:
451    """
452    Convert a MEDLINE XML file into plain text, one (metadata, text) pair per article.
453
454    Args:
455        source: path to the MEDLINE xml file
456        sections: which of "title"/"abstract" to include, and in what order.
457        passage_separator: string used to join the extracted passages into the single
458            returned text string.
459        clear_empty_brackets: see parse_pubmedxml.
460        fix_exponentials: see parse_pubmedxml.
461
462    Returns:
463        An iterator over one (PubMedMeta, text) pair per article
464    """
465    for pm_doc in parse_pubmedxml(
466        source, clear_empty_brackets=clear_empty_brackets, fix_exponentials=fix_exponentials
467    ):
468        text = passage_separator.join(pm_doc.iter_text(sections))
469        yield _pubmed_article_meta(pm_doc), text

472def pubmedxml2tagged(
473    source: Union[str, TextIO],
474    sections: Iterable[str] = ("title", "abstract"),
475    passage_separator: str = "\n\n",
476    keep_tags=PUBMED_KEEP_TAGS,
477) -> Iterator[Tuple[PubMedMeta, str]]:
478    """
479    Convert a MEDLINE XML file into marked-up text, one (metadata, text) pair per article, with
480    formatting tags (e.g. `<i>`, `<sup>`) kept inline instead of stripped. A thin wrapper around
481    parse_pubmedxml with return_xml=True and keep_tags defaulted to PUBMED_KEEP_TAGS.
482    clear_empty_brackets and fix_exponentials are always off - this is meant to keep markup
483    intact rather than clean up the text, and fix_exponentials has no effect under
484    return_xml=True anyway (see parse_pubmedxml).
485
486    Args:
487        source: path to the MEDLINE xml file
488        sections: which of "title"/"abstract" to include, and in what order.
489        passage_separator: string used to join the extracted passages into the single
490            returned text string.
491        keep_tags: see parse_pubmedxml. Defaults to `pubmed_constants.PUBMED_KEEP_TAGS`.
492
493    Returns:
494        An iterator over one (PubMedMeta, marked-up text) pair per article
495    """
496    for pm_doc in parse_pubmedxml(
497        source,
498        return_xml=True,
499        keep_tags=keep_tags,
500        clear_empty_brackets=False,
501        fix_exponentials=False,
502    ):
503        text = passage_separator.join(pm_doc.iter_text(sections))
504        yield _pubmed_article_meta(pm_doc), text

123@dataclass
124class PubMedArticle:
125    """One MEDLINE/PubMed article, as extracted by `parse_pubmedxml`."""
126
127    pmid: str
128    """PubMed ID."""
129
130    pmcid: Optional[str]
131    """PubMed Central ID, or None if not linked."""
132
133    doi: Optional[str]
134    """DOI, or None if not found."""
135
136    pub_year: Optional[int]
137    """Publication year, or None if not found."""
138
139    pub_month: Optional[int]
140    """Publication month, or None if not found."""
141
142    pub_day: Optional[int]
143    """Publication day, or None if not found."""
144
145    title: str
146    """Article title."""
147
148    abstract: Iterable[str]
149    """Abstract passages, one per `<AbstractText>` element."""
150
151    journal: str
152    """Journal title, or an empty string if not found."""
153
154    journal_iso: str
155    """ISO abbreviation of the journal title, or an empty string if not found."""
156
157    authors: Iterable[str]
158    """Author names, in document order."""
159
160    chemicals: Iterable[Chemical]
161    """Chemical substances, in document order."""
162
163    mesh_headings: Iterable[MeshHeading]
164    """MeSH headings, each with its own qualifiers nested inside, in document order."""
165
166    supplementary_mesh: Iterable[SupplementaryMeshConcept]
167    """Supplementary MeSH concepts, in document order."""
168
169    publication_types: Iterable[PublicationType]
170    """Publication types, in document order (excludes generic NLM support-type labels
171    like "Research Support, N.I.H., Extramural")."""
172
173    def iter_text(self, sections: Iterable[str] = ("title", "abstract")) -> Iterator[str]:
174        """
175        Yield each non-empty passage of text from the given fields, in order.
176
177        Args:
178            sections: which fields to pull text from, and in what order. Defaults to
179                ("title", "abstract").
180        """
181        for section in sections:
182            value = getattr(self, section)
183            texts = (value,) if isinstance(value, str) else value
184            for text in texts:
185                if text:
186                    yield text

 76@dataclass
 77class PubMedMeta:
 78    """Metadata for a MEDLINE/PubMed article - the text-free subset of `PubMedArticle`'s
 79    fields (everything but title/abstract), as returned alongside the text by
 80    `pubmedxml2txt`/`pubmedxml2tagged`."""
 81
 82    pmid: str
 83    """PubMed ID."""
 84
 85    pmcid: Optional[str]
 86    """PubMed Central ID, or None if not linked."""
 87
 88    doi: Optional[str]
 89    """DOI, or None if not found."""
 90
 91    pub_year: Optional[int]
 92    """Publication year, or None if not found."""
 93
 94    pub_month: Optional[int]
 95    """Publication month, or None if not found."""
 96
 97    pub_day: Optional[int]
 98    """Publication day, or None if not found."""
 99
100    journal: str
101    """Journal title, or an empty string if not found."""
102
103    journal_iso: str
104    """ISO abbreviation of the journal title, or an empty string if not found."""
105
106    authors: Iterable[str]
107    """Author names, in document order."""
108
109    chemicals: Iterable[Chemical]
110    """Chemical substances, in document order."""
111
112    mesh_headings: Iterable[MeshHeading]
113    """MeSH headings, each with its own qualifiers nested inside, in document order."""
114
115    supplementary_mesh: Iterable[SupplementaryMeshConcept]
116    """Supplementary MeSH concepts, in document order."""
117
118    publication_types: Iterable[PublicationType]
119    """Publication types, in document order (excludes generic NLM support-type labels
120    like "Research Support, N.I.H., Extramural")."""

 6@dataclass
 7class Chemical:
 8    """One `<Chemical>` entry from the article's ChemicalList."""
 9
10    ui: str
11    """MeSH substance ID (e.g. "D000068877")."""
12
13    name: str
14    """Substance name."""
15
16    registry_number: str
17    """CAS registry number, or "0" if none is assigned."""

34@dataclass
35class MeshHeading:
36    """One `<MeshHeading>` entry: a descriptor plus any qualifiers refining it."""
37
38    ui: str
39    """MeSH descriptor ID (e.g. "D009369")."""
40
41    name: str
42    """Descriptor name (e.g. "Neoplasms")."""
43
44    major_topic: bool
45    """Whether the descriptor itself is a major topic (MajorTopicYN="Y")."""
46
47    qualifiers: Iterable[MeshQualifier]
48    """Qualifiers refining this descriptor, in document order."""

20@dataclass
21class MeshQualifier:
22    """One `<QualifierName>` attached to a MeSH heading's descriptor."""
23
24    ui: str
25    """MeSH qualifier ID (e.g. "Q000378")."""
26
27    name: str
28    """Qualifier name (e.g. "genetics")."""
29
30    major_topic: bool
31    """Whether this qualifier is a major topic of the article (MajorTopicYN="Y")."""

51@dataclass
52class SupplementaryMeshConcept:
53    """One `<SupplMeshName>` entry from the article's SupplMeshList."""
54
55    ui: str
56    """Supplementary concept ID (e.g. "C000657245")."""
57
58    type: str
59    """One of "Disease", "Protocol", "Organism", "Anatomy", "Population" (DTD-enumerated)."""
60
61    name: str
62    """Concept name."""

65@dataclass
66class PublicationType:
67    """One `<PublicationType>` entry from the article's PublicationTypeList."""
68
69    ui: str
70    """MeSH publication-type ID (e.g. "D016428")."""
71
72    name: str
73    """Publication type name (e.g. "Journal Article")."""