<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "journalpublishing.dtd"><article xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:xlink="http://www.w3.org/1999/xlink" dtd-version="2.0" xml:lang="en" article-type="research-article"><front><journal-meta><journal-id journal-id-type="nlm-ta">JMIR Cardio</journal-id><journal-id journal-id-type="publisher-id">cardio</journal-id><journal-id journal-id-type="index">26</journal-id><journal-title>JMIR Cardio</journal-title><abbrev-journal-title>JMIR Cardio</abbrev-journal-title><issn pub-type="epub">2561-1011</issn><publisher><publisher-name>JMIR Publications</publisher-name><publisher-loc>Toronto, Canada</publisher-loc></publisher></journal-meta><article-meta><article-id pub-id-type="publisher-id">v10i1e87424</article-id><article-id pub-id-type="doi">10.2196/87424</article-id><article-categories><subj-group subj-group-type="heading"><subject>Original Paper</subject></subj-group></article-categories><title-group><article-title>Comparison of Rule-Based Natural Language Processing and Diagnosis Code Approaches to Detecting Bleeding in the Electronic Health Record: Algorithm Validation Within a Retrospective Cohort Study</article-title></title-group><contrib-group><contrib contrib-type="author" corresp="yes"><name name-style="western"><surname>Draper</surname><given-names>Evan W</given-names></name><degrees>PharmD</degrees><xref ref-type="aff" rid="aff1">1</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>Anderson</surname><given-names>Jan A</given-names></name><degrees>PharmD</degrees><xref ref-type="aff" rid="aff1">1</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>Shah</surname><given-names>Surbhi</given-names></name><degrees>MBBS</degrees><xref ref-type="aff" rid="aff2">2</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>Brown</surname><given-names>Leilia</given-names></name><degrees>MSN, RN</degrees><xref ref-type="aff" rid="aff3">3</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>Hanson</surname><given-names>Margaret R</given-names></name><degrees>RN</degrees><xref ref-type="aff" rid="aff4">4</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>Negley</surname><given-names>Kristin D</given-names></name><degrees>MS, APRN, CNS</degrees><xref ref-type="aff" rid="aff5">5</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>Lutz</surname><given-names>Jennifer M</given-names></name><degrees>MSN, RN</degrees><xref ref-type="aff" rid="aff6">6</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>Anderson</surname><given-names>Andrea E</given-names></name><degrees>BSPharm</degrees><xref ref-type="aff" rid="aff1">1</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>O'Meara</surname><given-names>John G</given-names></name><degrees>PharmD</degrees><xref ref-type="aff" rid="aff1">1</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>Jakowenko</surname><given-names>Nicholas D</given-names></name><degrees>PharmD</degrees><xref ref-type="aff" rid="aff1">1</xref></contrib><contrib contrib-type="author"><name name-style="western"><surname>McBane</surname><given-names>Robert D</given-names></name><degrees>MD</degrees><xref ref-type="aff" rid="aff7">7</xref></contrib></contrib-group><aff id="aff1"><institution>Department of Pharmacy, Mayo Clinic</institution><addr-line>201 1st St SW</addr-line><addr-line>Rochester</addr-line><addr-line>MN</addr-line><country>United States</country></aff><aff id="aff2"><institution>Division of Hematology and Medical Oncology, Department of Internal Medicine, Mayo Clinic</institution><addr-line>Phoenix</addr-line><addr-line>AZ</addr-line><country>United States</country></aff><aff id="aff3"><institution>Department of Nursing, Mayo Clinic</institution><addr-line>Jacksonville</addr-line><addr-line>FL</addr-line><country>United States</country></aff><aff id="aff4"><institution>Department of Nursing, Mayo Clinic Health System</institution><addr-line>Mankato</addr-line><addr-line>MN</addr-line><country>United States</country></aff><aff id="aff5"><institution>Department of Nursing, Mayo Clinic</institution><addr-line>Rochester</addr-line><addr-line>MN</addr-line><country>United States</country></aff><aff id="aff6"><institution>Department of Nursing, Mayo Clinic</institution><addr-line>Scottsdale</addr-line><addr-line>AZ</addr-line><country>United States</country></aff><aff id="aff7"><institution>Division of Vascular Medicine, Gonda Vascular Center, Department of Cardiovascular Medicine, Mayo Clinic</institution><addr-line>Rochester</addr-line><addr-line>MN</addr-line><country>United States</country></aff><contrib-group><contrib contrib-type="editor"><name name-style="western"><surname>Coristine</surname><given-names>Andrew</given-names></name></contrib></contrib-group><contrib-group><contrib contrib-type="reviewer"><name name-style="western"><surname>Le-Pogam</surname><given-names>Marie-Annick</given-names></name></contrib><contrib contrib-type="reviewer"><name name-style="western"><surname>Ananiadou</surname><given-names>Sophia</given-names></name></contrib></contrib-group><author-notes><corresp>Correspondence to Evan W Draper, PharmD, Department of Pharmacy, Mayo Clinic, 201 1st St SW, Rochester, MN, 55902, United States; <email>draper.evan@mayo.edu</email></corresp></author-notes><pub-date pub-type="collection"><year>2026</year></pub-date><pub-date pub-type="epub"><day>20</day><month>11</month><year>2026</year></pub-date><volume>10</volume><elocation-id>e87424</elocation-id><history><date date-type="received"><day>09</day><month>11</month><year>2025</year></date><date date-type="rev-recd"><day>23</day><month>09</month><year>2026</year></date><date date-type="accepted"><day>24</day><month>09</month><year>2026</year></date></history><copyright-statement>&#x00A9; Evan W Draper, Jan A Anderson, Surbhi Shah, Leilia Brown, Margaret R Hanson, Kristin D Negley, Jennifer M Lutz, Andrea E Anderson, John G O'Meara, Nicholas D Jakowenko, Robert D McBane. Originally published in JMIR Cardio (<ext-link ext-link-type="uri" xlink:href="https://cardio.jmir.org">https://cardio.jmir.org</ext-link>), 20.11.2026. </copyright-statement><copyright-year>2026</copyright-year><license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/"><p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (<ext-link ext-link-type="uri" xlink:href="https://creativecommons.org/licenses/by/4.0/">https://creativecommons.org/licenses/by/4.0/</ext-link>), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIR Cardio, is properly cited. The complete bibliographic information, a link to the original publication on <ext-link ext-link-type="uri" xlink:href="https://cardio.jmir.org">https://cardio.jmir.org</ext-link>, as well as this copyright and license information must be included.</p></license><self-uri xlink:type="simple" xlink:href="https://cardio.jmir.org/2026/1/e87424"/><abstract><sec><title>Background</title><p>Anticoagulant use is prevalent and increasing. High-quality anticoagulation management balances individual thrombotic and bleeding risks to optimize care. An automated algorithm to accurately extract major bleeding events from the electronic health record (EHR) of a large health care system would be extremely useful for tracking outcomes of anticoagulated patients.</p></sec><sec><title>Objective</title><p>The study aims to assess the accuracy of a natural language processing (NLP) approach, compared with and in conjunction with a diagnosis code (Dx Code)&#x2013;based strategy, for detecting major bleeding from the EHR.</p></sec><sec sec-type="methods"><title>Methods</title><p>Patients with at least 1 prescription for apixaban, dabigatran, edoxaban, rivaroxaban, or warfarin between January 1, 2019, and December 31, 2023, were analyzed for major bleeding events. Major bleeding was defined similarly to the International Society on Thrombosis and Haemostasis definition. For the NLP algorithm, clinical notes and radiology reports were screened for mentions of bleeding. MedTagger, an open-source, rule-based NLP tool, was then used on the notes. The NLP results were transformed into a feature set, which was used to create a support vector machine model. Three NLP models were created for central nervous system bleeding, critical site bleeding, and bleeding with a hemoglobin drop or transfusions (other major bleeding). Training and test datasets were created using an active learning approach. The model was trained using 80% of the dataset, while the remaining 20% was used for testing. After model finalization, an independent random sample was used to validate the model. Dx Codes were also used to detect major bleeding, similar to previously published approaches. Five unique algorithms were compared: (1) NLP, (2) Dx Code in the primary position, (3) Dx Code in any position, (4) NLP&#x2229;Dx Code in any position, and (5) NLP&#x222A;Dx Code in any position. All 5 algorithms were compared with manual chart review.</p></sec><sec sec-type="results"><title>Results</title><p>The final analysis identified 111,338 patients prescribed outpatient anticoagulation during the study. Manual adjudication was performed for 1189 of the 111,338 (1%) patients. Most of the 5 approaches performed well on the test data, with <italic>F</italic><sub>1</sub>-scores ranging from 0.545 to 0.959, except for Dx Code&#x2013;Primary, with an <italic>F</italic><sub>1</sub>-score of 0.353 for critical site bleeding. The <italic>F</italic><sub>1</sub>-scores for the validation data ranged from 0.509 to 0.679. The NLP&#x2229;Dx Code in any position approach yielded excellent precision of 0.864 (95% CI 0.667&#x2010;0.953) but only 0.559 (95% CI 0.395&#x2010;0.711) sensitivity. The NLP&#x222A;Dx Code in any position approach yielded good sensitivity of 0.824 (95% CI 0.665&#x2010;0.917) but only 0.368 (95% CI 0.269&#x2010;0.481) precision.</p></sec><sec sec-type="conclusions"><title>Conclusions</title><p>Electronic tools can screen the EHRs to identify major bleeding outcomes in large patient cohorts within a multisite health care network with high precision but only moderate sensitivity. Additional work is needed to enhance sensitivity while maintaining high precision.</p></sec></abstract><kwd-group><kwd>anticoagulants</kwd><kwd>hemorrhage</kwd><kwd>machine learning algorithms</kwd><kwd>natural language processing</kwd><kwd>population surveillance</kwd></kwd-group></article-meta></front><body><sec id="s1" sec-type="intro"><title>Introduction</title><p>A growing number of patients take antithrombotic therapy, including more than 8 million on anticoagulants in the United States and more than 25 million on antiplatelet agents [<xref ref-type="bibr" rid="ref1">1</xref>,<xref ref-type="bibr" rid="ref2">2</xref>]. Overtime, this number is only anticipated to grow with the aging population. These medications carry a risk of major hemorrhage, which adversely impacts quality of life and increases mortality risk. There are an estimated 250,000 anticoagulant-related emergency department (ED) visits annually for bleeding outcomes and for every 100 patients dispensed an anticoagulant, there are nearly 9 ED visits [<xref ref-type="bibr" rid="ref3">3</xref>]. High-quality anticoagulation delivery requires individualized patient assessment that balances thrombotic and bleeding risks to optimize management. This decision-making process is nuanced and dynamic, with overlap between bleeding and thrombotic risk factors as patients age.</p><p>High-quality delivery of anticoagulation for large health care networks involving tens of thousands of patients can be challenging. When considering ideal oversight of large anticoagulation networks, tools capable of monitoring hard outcomes are essential for quality improvement, and central to this process is an accurate and near-real&#x2013;time assessment of bleeding events.</p><p>Given the widespread use of antithrombotic agents, manual chart abstraction for bleeding outcomes ascertainment is neither timely, cost-effective, nor practical. Methodologies using diagnosis code (Dx Code)&#x2013;based major bleeding detection are widely used [<xref ref-type="bibr" rid="ref4">4</xref>-<xref ref-type="bibr" rid="ref8">8</xref>] but may lack sensitivity, specificity, and overall accuracy [<xref ref-type="bibr" rid="ref9">9</xref>-<xref ref-type="bibr" rid="ref11">11</xref>]. Electronic tools capable of automated and near-real&#x2013;time identification of bleeding events could facilitate quality improvement and the refinement of safer anticoagulation delivery.</p><p>Established in 2019, the Mayo Clinic Anticoagulation and Antiplatelet Stewardship Quality Committee has been charged with oversight of antithrombotic delivery across the multisite institution. Since 2019, this has included more than 115,000 patients receiving a direct oral anticoagulant and more than 40,000 patients treated with warfarin. To facilitate the management of this large patient population, tools capable of providing near-real&#x2013;time detection of bleeding and thrombosis are required. The objective of this study was to assess the accuracy of a natural language processing (NLP) approach compared with and in conjunction with a Dx Code&#x2013;based strategy for detecting major bleeding.</p></sec><sec id="s2" sec-type="methods"><title>Methods</title><sec id="s2-1"><title>Patient Population</title><p>An algorithm validation within a retrospective cohort study design was used to evaluate the accuracy of NLP, Dx Codes, and NLP plus Dx Codes to identify major bleeding in the electronic health record (EHR). Patients with at least one outpatient prescription for apixaban, dabigatran, edoxaban, rivaroxaban, or warfarin (January 1, 2019, to December 31, 2023) were included in the analysis. Data were collected from a centralized EHR that contains detailed information about all inpatient hospitalizations, outpatient visits, radiology images and reports, laboratory results, pathology results, and patient correspondence. The indication for anticoagulation was estimated based on Dx Codes in the EHR. Demographic information, indication, and prescribed medication were provided for all manually reviewed patients. Any missing values were categorized as unknown.</p></sec><sec id="s2-2"><title>Bleeding Definition</title><p>Major bleeding was defined similar to the ISTH (International Society on Thrombosis and Haemostasis) and included symptomatic bleeding into a critical area or organ, and bleeding causing a hemoglobin decrease of at least 2 g/dL or transfusions of at least 2 units of packed red blood cells (PRBC) [<xref ref-type="bibr" rid="ref12">12</xref>]. Symptomatic bleeding into a critical area or organ was divided into 2 categories: central nervous system (CNS) bleeding and critical site bleeding. CNS bleeds were defined as either intracranial or intraspinal bleeding. Critical site bleeding included intraocular, retroperitoneal, pericardial, hemothorax [<xref ref-type="bibr" rid="ref13">13</xref>,<xref ref-type="bibr" rid="ref14">14</xref>], and intra-articular bleeding. Bleeding with a hemoglobin drop or transfusions was referred to as &#x201C;other major bleeding.&#x201D; Fatal bleeding is included in the ISTH bleeding definition. However, a Dx Code specific to fatal bleeding does not exist, making it impossible to evaluate specifically for fatal bleeding without assuming the cause of death or manually reviewing the chart. Fatal bleeding events are likely to be found by the NLP approach and Dx Codes, but neither approach will specifically label the event as a fatal bleed. Mortality occurring within the same month as a bleeding event was collected and is described.</p><p>The ISTH definition of symptomatic bleeding with a 2 g/dL or greater change in hemoglobin or transfusion of 2 or more units of PRBC (&#x201C;other major bleeding&#x201D;) does not specify the timing for comparing hemoglobin results or for PRBC transfusions. For the purposes of this study, a qualifying drop in hemoglobin was defined by 2 methods. The first method compared a hemoglobin result with any other hemoglobin result within the preceding 30 days. The second method compared a hemoglobin result to the immediately prior hemoglobin result. If either method was true, then the hemoglobin change met the definition. Using both approaches captures a slow, occult bleed with months between hemoglobin results as well as a more acute drop in hemoglobin values. For the transfusion requirement definition, 2 or more units of PRBC had to be administered within a 3-day time increment. If there was a 4-day separation between transfusions, that would not meet the definition. Other studies have proposed a 48-hour limit on the hemoglobin change and did not specify a time frame for the transfusions [<xref ref-type="bibr" rid="ref15">15</xref>]. The temporal thresholds for transfusions and hemoglobin change for this study were determined through active learning and by examining bleeding events within the EHR.</p></sec><sec id="s2-3"><title>Event Identification and Adjudication</title><p>To compare the accuracy of NLP and Dx Codes to detect bleeding outcomes, manual chart review for event validation was performed. Positive bleeding events for training, testing, and validation were manually and independently reviewed in the EHR by at least 2 study team members (nurses, pharmacists, or physicians) to create the gold standard against which the NLP and Dx Code approaches were compared. Months in which no mentions of bleeding terms were noted throughout the EHR were assumed to be months without a bleeding event, and no EHR manual review was completed. Months in which bleeding terms were found or Dx Codes were present were manually reviewed in the EHR by 1 study team member. Obvious nonevents, such as &#x201C;no intracranial hemorrhage present,&#x201D; were categorized as nonevents by only 1 study team member. Ambiguous situations such as &#x201C;differential diagnosis includes hemarthrosis, septic arthritis&#x2026;&#x201D; were reviewed by at least 2 study team members when the initial review could not confidently rule out a bleeding event. The date and type of the bleed (CNS, critical site, or other major bleed) were recorded, and each month adjudicated was classified as a bleeding event or not a bleeding event. Adjudication by month was chosen after an initial 3-day rolling window proved performance-intensive. Cohen &#x03BA; was calculated by evaluating interteam member rating reliability on the validation sample. Specific sentences or notes were not annotated for NLP purposes. A bleeding event could occur in the outpatient or inpatient setting. Event disagreements were resolved through group discussion. Only the incident bleeding event was included for patients with multiple events. While prescribed anticoagulation was an inclusion criterion for the analysis, temporary anticoagulant interruption prior to the bleeding event did not exclude the patient. For event adjudication, surgical bleeding leading to a drop in hemoglobin or transfusion was excluded. The NLP algorithm does not have the functionality to exclude surgical bleeding, and future work would be required to align the algorithm with manual review. Additional details on event adjudication are available in the Section S1 of <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>.</p></sec><sec id="s2-4"><title>NLP</title><p>MedTagger is an open-source, rule-based NLP program from Open Health Natural Language Processing (OHNLP) Consortium that detects specific terms and then uses more than 700 rules to determine the context of each term [<xref ref-type="bibr" rid="ref16">16</xref>]. MedTagger classifies the context using the following elements: experiencer: patient or other individual; status: present or historical; and level of certainty: positive, negative, possible, or hypothetical. The text was not preprocessed before using MedTagger. A MedTagger version based on the OHNLP Toolkit (v1.0.82) was used.</p><p>All patient notes were searched for CNS bleeding and critical site bleeding using regular expressions. The incident date and 3 days following a 2 g/dL or greater drop in hemoglobin or 2 or more units of PRBC were also searched using regular expressions. Search terms were specific to each bleeding type. Additional information on the search terms is provided in Section S2 of <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>. Notes with bleeding mentions found by regular expressions were further processed with MedTagger NLP. Exclusionary terms were added to the algorithm for NLP refinement. Additional details on MedTagger search terms and excluded note sections are available in Section S3 of <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>.</p><p>A feature set for each algorithm was created, and machine learning using a support vector machine model was created to identify bleeds. Support vector machine was selected as the model after evaluating logistic regression, random forest, and gradient-boosted machine models. Adjudicated data were divided into training (80%) and testing sets (20%), stratified by patient to ensure no data-leakage. Months without a bleeding event were combined with months with a CNS bleed and used to create the CNS bleed model. The same months without a bleeding event were used to create the critical site bleed and other major bleed models. The feature set aggregated NLP results by month, and NLP results from clinical notes and radiology notes were handled as separate features. Machine learning parameters were chosen using grid search and cross-validation on the training set. Additional information on model training is provided in Section S4 of <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>. A visual representation of the approach is provided in <xref ref-type="fig" rid="figure1">Figure 1</xref>.</p><fig position="float" id="figure1"><label>Figure 1.</label><caption><p>MedTagger process. MedTagger is an open-source, rule-based natural language processing (NLP) tool that categorizes medical terms with regard to certainty (ie, positive, negated, or hypothetical) and status (ie, present or history of). This work used MedTagger to categorize mentions of bleeding found in clinical and radiology notes in an attempt to find major bleeding events among patients on anticoagulation. The MedTagger results are aggregated to create a feature set, which is subsequently used in machine learning. MRN: medical record number; *each combination of NLP results creates an additional feature; **one row per MRN per month.</p></caption><graphic alt-version="no" mimetype="image" position="float" xlink:type="simple" xlink:href="cardio_v10i1e87424_fig01.png"/></fig><p>The NLP approach was refined iteratively by reviewing high and low probability cases similar to active learning (<xref ref-type="fig" rid="figure2">Figure 2</xref>) [<xref ref-type="bibr" rid="ref17">17</xref>]. This approach was used to select cases for manual review and adjudication during model development. This approach does not suffer as much from class imbalance compared to random sampling. After model development and finalization, an independent random sample of patients was manually adjudicated and run through the model.</p><fig position="float" id="figure2"><label>Figure 2.</label><caption><p>Natural language processing (NLP) model development. This retrospective cohort study of Mayo Clinic patients prescribed oral anticoagulation from January 1, 2019, to December 31, 2023, assessed the feasibility and accuracy of an NLP and machine learning (ML) approach to detect major bleeding from the electronic health record (EHR). The model development process iteratively refined the search terms, the method of combining discrete hemoglobin and transfusion data, and the feature creation process. The finalized NLP model was applied to an independent, random sample and was compared with 2 different diagnosis code approaches.</p></caption><graphic alt-version="no" mimetype="image" position="float" xlink:type="simple" xlink:href="cardio_v10i1e87424_fig02.png"/></fig></sec><sec id="s2-5"><title>Dx Code</title><p>Bleeding events were also identified using Dx Codes [<xref ref-type="bibr" rid="ref8">8</xref>]. Dx Codes associated with ED visits or hospital admissions were included. The primary Dx Code (&#x201C;Dx Code&#x2013;Primary&#x201D;) or any Dx Code associated with the encounter (&#x201C;Dx Code&#x2013;Any Position&#x201D;) were evaluated. Specific <italic>ICD-10</italic> (<italic>International Statistical Classification of Diseases, Tenth Revision</italic>) codes used are provided in Section S5 of <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>. Data were aggregated by month based on the admission date.</p></sec><sec id="s2-6"><title>NLP Combined With Dx Code</title><p>The Dx Code&#x2013;Any Position approach and the NLP approach were combined in 2 ways: &#x201C;NLP&#x2229;Dx Code Any&#x201D; requires both the NLP and Dx Code&#x2013;Any Position results to be positive, while &#x201C;NLP&#x222A;Dx Code Any&#x201D; allows either the NLP or the Dx Code&#x2013;Any Position approach to be positive.</p></sec><sec id="s2-7"><title>Algorithm and Comparison Summary</title><p>Five approaches for three different bleeding types were created: (1) NLP, (2) Dx Code&#x2013;Primary, (3) Dx Code&#x2013;Any, (4) NLP&#x2229;Dx Code Any, and (5) NLP&#x222A;Dx Code Any. For model test data, the 5 approaches were evaluated for each of the 3 different bleeding types (CNS bleeding, critical site bleeding, and other major bleeding). For model validation on a random sample of data, the 3 bleeding types were combined due to the low volume of events.</p></sec><sec id="s2-8"><title>Statistical Analysis</title><p>Accuracy measures of sensitivity, specificity, precision, and <italic>F</italic><sub>1</sub>-score, with 95% CIs, were calculated for each approach compared against the gold standard of a manually adjudicated review. The methodology to calculate the 95% CIs was SciPy&#x2019;s implementation of the Wilson without continuity correction for sensitivity, specificity, and precision [<xref ref-type="bibr" rid="ref18">18</xref>]. Given no clear best approach for calculating a CI for <italic>F</italic><sub>1</sub>-score [<xref ref-type="bibr" rid="ref19">19</xref>], a bootstrap methodology using 5000 samples was used to calculate the 95% CI for <italic>F</italic><sub>1</sub>-score. The <italic>F</italic><sub>1</sub>-score is a measurement that evaluates both sensitivity and precision in a single metric. Cohen &#x03BA; was calculated by evaluating interteam member rating reliability and used the bootstrap methodology. The above analyses were performed for each type of bleeding as well as for a combined &#x201C;all bleeding&#x201D; analysis. All analyses were performed using Python (version 3.10.18).</p></sec><sec id="s2-9"><title>Ethical Considerations</title><p>The project was deemed exempt by the Mayo Clinic Institutional Review Board in the context of a quality improvement project. Patient information was stored in a secure environment within the Mayo Clinic Cloud. Study team members were provided with access to the minimum amount of identifiable information necessary to verify the bleeding event during event adjudication.</p></sec></sec><sec id="s3" sec-type="results"><title>Results</title><p>The final analysis identified 111,338 total patients prescribed outpatient anticoagulation over the study time frame. Clinical notes and radiology reports for the included patients were searched for bleeding terms, identifying 264,551 clinical notes and radiology reports for CNS bleeds, 73,973 for critical site bleeds, and 581,120 for other major bleeding. The total number of unique notes with bleeding terms found is 773,779. These notes and reports were processed using MedTagger (<xref ref-type="fig" rid="figure3">Figure 3</xref>). A cohort of 604 patients was manually adjudicated during active learning and was used for model training and testing. Each patient and month reviewed contributed to model development. A total of 1001 patient-months were manually reviewed, with 643 months without a bleeding event, 183 months with CNS bleeding, 68 months with critical site bleeding, and 107 months with other major bleeding. An additional random sample of 585 patients was used to validate the final model. The validation sample yielded a total of 30,757 patient-months, with 30,723 months without a bleeding event, 7 months with a CNS bleed, 7 months with a critical site bleed, and 20 months with other major bleed. Cohen &#x03BA; between the reviewers for the random sample was 0.66 (95% CI 0.38-0.88). Demographic information, indications, and medications prescribed for all patients manually reviewed are provided in <xref ref-type="table" rid="table1">Table 1</xref>. The patients had a mean age of 70 (SD 14.4) years, and 57% were male patients. The majority of patients were White individuals. The 2 most common indications for anticoagulant therapy were atrial fibrillation and thromboembolism. The 2 most common anticoagulants used were apixaban and warfarin.</p><fig position="float" id="figure3"><label>Figure 3.</label><caption><p>Pipeline and adjudicated reviews flow diagram. A natural language processing (NLP) and machine learning&#x2013;based approach to detect major bleeding events from the electronic health record (EHR) was developed and evaluated. The approach processed 111,338 patients and 773,779 unique notes. Patients on oral anticoagulation were first identified. Notes mentioning bleeding were found and processed by MedTagger. Manual reviews of bleeding events were performed to evaluate the accuracy of the approach. CNS: central nervous system.</p></caption><graphic alt-version="no" mimetype="image" position="float" xlink:type="simple" xlink:href="cardio_v10i1e87424_fig03.png"/></fig><table-wrap id="t1" position="float"><label>Table 1.</label><caption><p>Baseline demographics and clinical characteristics.</p></caption><table id="table1" frame="hsides" rules="groups"><thead><tr><td align="left" valign="bottom">Characteristic</td><td align="left" valign="bottom">Patients (N=1189)</td></tr></thead><tbody><tr><td align="left" valign="top">Age (y), mean (SD)</td><td align="left" valign="top">70.0 (14.4)</td></tr><tr><td align="left" valign="top">Sex: female, n (%)</td><td align="left" valign="top">514 (43.2)</td></tr><tr><td align="left" valign="top" colspan="2">Race, n (%)</td></tr><tr><td align="left" valign="top">&#x2003;&#x2003;White</td><td align="left" valign="top">1108 (93.2)</td></tr><tr><td align="left" valign="top">&#x2003;&#x2003;Black</td><td align="left" valign="top">29 (2.4)</td></tr><tr><td align="left" valign="top">&#x2003;&#x2003;Asian</td><td align="left" valign="top">20 (1.7)</td></tr><tr><td align="left" valign="top">&#x2003;&#x2003;American Indian/Alaskan Native</td><td align="left" valign="top">5 (0.4)</td></tr><tr><td align="left" valign="top">&#x2003;&#x2003;Other/unknown</td><td align="left" valign="top">27 (2.3)</td></tr><tr><td align="left" valign="top" colspan="2">Ethnicity, n (%)</td></tr><tr><td align="left" valign="top">&#x2003;&#x2003;Not Hispanic/Latino</td><td align="left" valign="top">1142 (96.0)</td></tr><tr><td align="left" valign="top">&#x2003;&#x2003;Hispanic/Latino</td><td align="left" valign="top">25 (2.1)</td></tr><tr><td align="left" valign="top">&#x2003;&#x2003;Not disclosed/unknown</td><td align="left" valign="top">22 (1.9)</td></tr><tr><td align="left" valign="top" colspan="2">Anticoagulant indication, n (%)</td></tr><tr><td align="left" valign="top">&#x2003;&#x2003;Atrial fibrillation</td><td align="left" valign="top">638 (53.7)</td></tr><tr><td align="left" valign="top">&#x2003;&#x2003;Thromboembolism</td><td align="left" valign="top">399 (33.6)</td></tr><tr><td align="left" valign="top">&#x2003;&#x2003;Other/unknown</td><td align="left" valign="top">111 (9.3)</td></tr><tr><td align="left" valign="top">&#x2003;&#x2003;Heart valve prosthesis or repair</td><td align="left" valign="top">23 (1.9)</td></tr><tr><td align="left" valign="top">&#x2003;&#x2003;Peripheral artery disease</td><td align="left" valign="top">18 (1.5)</td></tr><tr><td align="left" valign="top" colspan="2">Anticoagulant used, n (%)</td></tr><tr><td align="left" valign="top">&#x2003;&#x2003;Apixaban</td><td align="left" valign="top">630 (53.0)</td></tr><tr><td align="left" valign="top">&#x2003;&#x2003;Warfarin</td><td align="left" valign="top">342 (28.8)</td></tr><tr><td align="left" valign="top">&#x2003;&#x2003;Rivaroxaban</td><td align="left" valign="top">208 (17.5)</td></tr><tr><td align="left" valign="top">&#x2003;&#x2003;Dabigatran</td><td align="left" valign="top">8 (0.7)</td></tr><tr><td align="left" valign="top">&#x2003;&#x2003;Edoxaban</td><td align="left" valign="top">1 (0.1)</td></tr></tbody></table></table-wrap><p>Performance metrics and 95% CIs for the testing data and individual bleeding types (<xref ref-type="table" rid="table2">Table 2</xref>) compare the performance of the 5 different approaches. CNS bleeding algorithms performed the best with <italic>F</italic><sub>1</sub>-scores of 0.946 (95% CI 0.885&#x2010;0.989) for NLP, 0.959 (95% CI 0.904&#x2010;1.000) for Dx Code&#x2013;Primary, and 0.947 (95% CI 0.886&#x2010;0.989) for Dx Code&#x2013;Any, 0.959 (95% CI 0.906&#x2010;1.000) for NLP&#x2229;Dx Code Any, and 0.935 (95% CI 0.868&#x2010;0.986) for NLP&#x222A;Dx Code Any.</p><table-wrap id="t2" position="float"><label>Table 2.</label><caption><p>Performance metrics for individual bleeding types for test data.</p></caption><table id="table2" frame="hsides" rules="groups"><thead><tr><td align="left" valign="bottom">Bleeding classification method</td><td align="left" valign="bottom">Sensitivity (95% CI)</td><td align="left" valign="bottom">Specificity (95% CI)</td><td align="left" valign="bottom">Precision (95% CI)</td><td align="left" valign="bottom"><italic>F</italic><sub>1</sub>-score (95% CI)</td></tr></thead><tbody><tr><td align="left" valign="top" colspan="5">CNS<sup><xref ref-type="table-fn" rid="table2fn1">a</xref></sup> bleeds</td></tr><tr><td align="left" valign="top"><named-content content-type="indent">&#x00A0;&#x00A0;&#x00A0;&#x00A0;</named-content>NLP<sup><xref ref-type="table-fn" rid="table2fn2">b</xref></sup></td><td align="left" valign="top">0.946 (0.823&#x2010;0.985)</td><td align="left" valign="top">0.982 (0.938&#x2010;0.995)</td><td align="left" valign="top">0.946 (0.823&#x2010;0.985)</td><td align="left" valign="top">0.946 (0.885&#x2010;0.989)</td></tr><tr><td align="left" valign="top"><named-content content-type="indent">&#x00A0;&#x00A0;&#x00A0;&#x00A0;</named-content>Dx Code<sup><xref ref-type="table-fn" rid="table2fn3">c</xref></sup>&#x2013;Primary</td><td align="left" valign="top">0.946 (0.823&#x2010;0.985)</td><td align="left" valign="top">0.991 (0.952&#x2010;0.998)</td><td align="left" valign="top">0.972 (0.858&#x2010;0.995)</td><td align="left" valign="top">0.959 (0.904&#x2010;1.000)</td></tr><tr><td align="left" valign="top"><named-content content-type="indent">&#x00A0;&#x00A0;&#x00A0;&#x00A0;</named-content>Dx Code&#x2013;Any</td><td align="left" valign="top">0.973 (0.862&#x2010;0.995)</td><td align="left" valign="top">0.974 (0.925&#x2010;0.991)</td><td align="left" valign="top">0.923 (0.797&#x2010;0.973)</td><td align="left" valign="top">0.947 (0.886&#x2010;0.989)</td></tr><tr><td align="left" valign="top"><named-content content-type="indent">&#x00A0;&#x00A0;&#x00A0;&#x00A0;</named-content>NLP&#x2229;Dx Code Any<sup><xref ref-type="table-fn" rid="table2fn4">d</xref></sup></td><td align="left" valign="top">0.946 (0.823&#x2010;0.985)</td><td align="left" valign="top">0.991 (0.952&#x2010;0.998)</td><td align="left" valign="top">0.972 (0.858&#x2010;0.995)</td><td align="left" valign="top">0.959 (0.906&#x2010;1.000)</td></tr><tr><td align="left" valign="top"><named-content content-type="indent">&#x00A0;&#x00A0;&#x00A0;&#x00A0;</named-content>NLP&#x222A;Dx Code Any<sup><xref ref-type="table-fn" rid="table2fn5">e</xref></sup></td><td align="left" valign="top">0.973 (0.862&#x2010;0.995)</td><td align="left" valign="top">0.965 (0.913&#x2010;0.986)</td><td align="left" valign="top">0.900 (0.769&#x2010;0.960)</td><td align="left" valign="top">0.935 (0.868&#x2010;0.986)</td></tr><tr><td align="left" valign="top" colspan="5">Critical site bleeds</td></tr><tr><td align="left" valign="top"><named-content content-type="indent">&#x00A0;&#x00A0;&#x00A0;&#x00A0;</named-content>NLP</td><td align="left" valign="top">0.643 (0.388&#x2010;0.837)</td><td align="left" valign="top">0.983 (0.942&#x2010;0.995)</td><td align="left" valign="top">0.818 (0.523&#x2010;0.949)</td><td align="left" valign="top">0.720 (0.471&#x2010;0.897)</td></tr><tr><td align="left" valign="top"><named-content content-type="indent">&#x00A0;&#x00A0;&#x00A0;&#x00A0;</named-content>Dx Code&#x2013;Primary</td><td align="left" valign="top">0.214 (0.076&#x2010;0.476)</td><td align="left" valign="top">1.000 (0.969&#x2010;1.000)</td><td align="left" valign="top">1.000 (0.439&#x2010;1.000)</td><td align="left" valign="top">0.353 (0.000&#x2010;0.632)</td></tr><tr><td align="left" valign="top"><named-content content-type="indent">&#x00A0;&#x00A0;&#x00A0;&#x00A0;</named-content>Dx Code&#x2013;Any</td><td align="left" valign="top">0.429 (0.214&#x2010;0.674)</td><td align="left" valign="top">0.983 (0.942&#x2010;0.995)</td><td align="left" valign="top">0.750 (0.409&#x2010;0.929)</td><td align="left" valign="top">0.545 (0.235&#x2010;0.769)</td></tr><tr><td align="left" valign="top"><named-content content-type="indent">&#x00A0;&#x00A0;&#x00A0;&#x00A0;</named-content>NLP&#x2229;Dx Code Any</td><td align="left" valign="top">0.429 (0.214&#x2010;0.674)</td><td align="left" valign="top">1.000 (0.969&#x2010;1.000)</td><td align="left" valign="top">1.000 (0.610&#x2010;1.000)</td><td align="left" valign="top">0.600 (0.286&#x2010;0.828)</td></tr><tr><td align="left" valign="top"><named-content content-type="indent">&#x00A0;&#x00A0;&#x00A0;&#x00A0;</named-content>NLP&#x222A;Dx Code Any</td><td align="left" valign="top">0.643 (0.388&#x2010;0.837)</td><td align="left" valign="top">0.967 (0.918&#x2010;0.987)</td><td align="left" valign="top">0.692 (0.424&#x2010;0.873)</td><td align="left" valign="top">0.667 (0.421&#x2010;0.857)</td></tr><tr><td align="left" valign="top" colspan="5">Other major bleeds</td></tr><tr><td align="left" valign="top"><named-content content-type="indent">&#x00A0;&#x00A0;&#x00A0;&#x00A0;</named-content>NLP</td><td align="left" valign="top">0.818 (0.615&#x2010;0.927)</td><td align="left" valign="top">0.958 (0.906&#x2010;0.982)</td><td align="left" valign="top">0.783 (0.581&#x2010;0.903)</td><td align="left" valign="top">0.800 (0.650&#x2010;0.913)</td></tr><tr><td align="left" valign="top"><named-content content-type="indent">&#x00A0;&#x00A0;&#x00A0;&#x00A0;</named-content>Dx Code&#x2013;Primary</td><td align="left" valign="top">0.545 (0.347&#x2010;0.731)</td><td align="left" valign="top">0.975 (0.929&#x2010;0.991)</td><td align="left" valign="top">0.800 (0.548&#x2010;0.930)</td><td align="left" valign="top">0.649 (0.437&#x2010;0.809)</td></tr><tr><td align="left" valign="top"><named-content content-type="indent">&#x00A0;&#x00A0;&#x00A0;&#x00A0;</named-content>Dx Code&#x2013;Any</td><td align="left" valign="top">0.636 (0.43&#x2010;0.803)</td><td align="left" valign="top">0.942 (0.884&#x2010;0.971)</td><td align="left" valign="top">0.667 (0.454&#x2010;0.828)</td><td align="left" valign="top">0.651 (0.452&#x2010;0.800)</td></tr><tr><td align="left" valign="top"><named-content content-type="indent">&#x00A0;&#x00A0;&#x00A0;&#x00A0;</named-content>NLP&#x2229;Dx Code Any</td><td align="left" valign="top">0.591 (0.387&#x2010;0.767)</td><td align="left" valign="top">0.983 (0.941&#x2010;0.995)</td><td align="left" valign="top">0.867 (0.621&#x2010;0.963)</td><td align="left" valign="top">0.703 (0.500&#x2010;0.851)</td></tr><tr><td align="left" valign="top"><named-content content-type="indent">&#x00A0;&#x00A0;&#x00A0;&#x00A0;</named-content>NLP&#x222A;Dx Code Any</td><td align="left" valign="top">0.864 (0.667&#x2010;0.953)</td><td align="left" valign="top">0.917 (0.853&#x2010;0.954)</td><td align="left" valign="top">0.655 (0.473&#x2010;0.801)</td><td align="left" valign="top">0.745 (0.595&#x2010;0.864)</td></tr></tbody></table><table-wrap-foot><fn id="table2fn1"><p><sup>a</sup>CNS: central nervous system.</p></fn><fn id="table2fn2"><p><sup>b</sup>NLP: natural language processing. </p></fn><fn id="table2fn3"><p><sup>c</sup>Dx Code: diagnosis code.</p></fn><fn id="table2fn4"><p><sup>d</sup>NLP&#x2229;Dx Code Any is the intersection (&#x2229;) of NLP and Dx Code Any, where both approaches must be positive to qualify as a bleed. </p></fn><fn id="table2fn5"><p><sup>e</sup>NLP&#x222A;Dx Code Any is the union (&#x222A;) of NLP and Dx Codes where either approach can be positive to qualify as a bleed.</p></fn></table-wrap-foot></table-wrap><p>Based on <italic>F</italic><sub>1</sub>-scores, the NLP approach performed the best for critical site bleeds, with 0.720 (95% CI 0.471&#x2010;0.897), and for other major bleeds, with a score of 0.800 (95% CI 0.650&#x2010;0.913).</p><p>Performance metrics for the validation data and combined bleeds (<xref ref-type="table" rid="table3">Table 3</xref>) demonstrate decreased performance compared with the test data. <italic>F</italic><sub>1</sub>-scores ranged from 0.509 (95% CI 0.388-0.618) to 0.679 (95% CI 0.524-0.812) for validation compared with up to 0.959 for the test data. The best precision was for the NLP&#x2229;Dx Code Any with 0.864 (95% CI 0.667&#x2010;0.953). However, this approach only yielded 0.559 (95% CI 0.395&#x2010;0.711) sensitivity. Model features are available in Section S6 of <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>.</p><table-wrap id="t3" position="float"><label>Table 3.</label><caption><p>Performance metrics for the random validation sample.</p></caption><table id="table3" frame="hsides" rules="groups"><thead><tr><td align="left" valign="bottom">Bleeding identification method</td><td align="left" valign="bottom">Sensitivity (95% CI)</td><td align="left" valign="bottom">Specificity (95% CI)</td><td align="left" valign="bottom">Precision (95% CI)</td><td align="left" valign="bottom"><italic>F</italic><sub>1</sub>-score (95% CI)</td></tr></thead><tbody><tr><td align="left" valign="top">NLP<sup><xref ref-type="table-fn" rid="table3fn1">a</xref></sup></td><td align="left" valign="top">0.676 (0.508&#x2010;0.809)</td><td align="left" valign="top">0.999 (0.999&#x2010;1.000)</td><td align="left" valign="top">0.511 (0.370&#x2010;0.650)</td><td align="left" valign="top">0.582 (0.441&#x2010;0.703)</td></tr><tr><td align="left" valign="top">Dx Code<sup><xref ref-type="table-fn" rid="table3fn2">b</xref></sup>&#x2013;Primary</td><td align="left" valign="top">0.618 (0.450&#x2010;0.761)</td><td align="left" valign="top">1.000 (0.999&#x2010;1.000)</td><td align="left" valign="top">0.618 (0.450&#x2010;0.761)</td><td align="left" valign="top">0.618 (0.462&#x2010;0.738)</td></tr><tr><td align="left" valign="top">Dx Code&#x2013;Any</td><td align="left" valign="top">0.706 (0.538&#x2010;0.832)</td><td align="left" valign="top">0.999 (0.999&#x2010;0.999)</td><td align="left" valign="top">0.453 (0.327&#x2010;0.585)</td><td align="left" valign="top">0.552 (0.415&#x2010;0.667)</td></tr><tr><td align="left" valign="top">NLP&#x2229;Dx Code Any<sup><xref ref-type="table-fn" rid="table3fn3">c</xref></sup></td><td align="left" valign="top">0.559 (0.395&#x2010;0.711)</td><td align="left" valign="top">1.000 (1.000&#x2010;1.000)<sup><xref ref-type="table-fn" rid="table3fn4">d</xref></sup></td><td align="left" valign="top">0.864 (0.667&#x2010;0.953)</td><td align="left" valign="top">0.679 (0.524&#x2010;0.812)</td></tr><tr><td align="left" valign="top">NLP&#x222A;Dx Code Any<sup><xref ref-type="table-fn" rid="table3fn5">e</xref></sup></td><td align="left" valign="top">0.824 (0.665&#x2010;0.917)</td><td align="left" valign="top">0.998 (0.998&#x2010;0.999)</td><td align="left" valign="top">0.368 (0.269&#x2010;0.481)</td><td align="left" valign="top">0.509 (0.388&#x2010;0.618)</td></tr></tbody></table><table-wrap-foot><fn id="table3fn1"><p><sup>a</sup>NLP: natural language processing.</p></fn><fn id="table3fn2"><p><sup>b</sup>Dx Code: diagnosis code.</p></fn><fn id="table3fn3"><p><sup>c</sup>NLP&#x2229;Dx Code Any is the intersection (&#x2229;) of NLP and Dx Code Any, where both approaches must be positive to qualify as a bleed.</p></fn><fn id="table3fn4"><p><sup>d</sup>The specificity, rounded to 6 decimal places, is 0.999902 (0.999713-0.999967). </p></fn><fn id="table3fn5"><p><sup>e</sup>NLP&#x222A;Dx Code Any is the union (&#x222A;) of NLP and Dx Code Any, where either approach can be positive to qualify as a bleed.</p></fn></table-wrap-foot></table-wrap><p>Mortality in the training and test data occurred within the same month as the bleed outcome for 22 out of 183 (12%) CNS bleeds, 2 out of 68 (3%) critical site bleeds, and 5 out of 107 (5%) other major bleeds. Mortality in the validation data occurred within the same month, 2 out of 7 (29%) of CNS bleeds, 0 out of 7 (0%) critical site bleeds, and 2 out of 20 (10%) other major bleeds. While causality assignment for fatal bleeding can be challenging, the temporal association between the bleeding event and death suggests a potential contribution to this outcome.</p></sec><sec id="s4" sec-type="discussion"><title>Discussion</title><sec id="s4-1"><title>Principal Findings</title><p>The principal finding of the current study is that electronic tools can screen electronic medical records for major bleeding outcomes from large patient cohorts within a multisite health care network with high precision and moderate sensitivity. In validation, the NLP&#x2229;Dx Code Any approach may have the most clinical utility with the highest precision of 0.864 (95% CI 0.667&#x2010;0.953) and moderate sensitivity of 0.559 (95% CI 0.395&#x2010;0.711). Given the high volume of patients on anticoagulant medications, an electronic tool that maximizes precision might provide the most useful measure for groups focused on quality improvement and process oversight.</p><p>The clinical utility of these findings is notable for several reasons. One could envision the use of this information to improve patient outcomes. First, appropriate use criteria could be assessed to ensure anticoagulants are appropriately prescribed. For example, patients with severe renal impairment, liver impairment, or thrombocytopenia may be at particular risk for bleeding outcomes. When tracked electronically, &#x201C;our practice alerts&#x201D; could then be delivered to providers. Second, duplicate antithrombotic delivery could be scrutinized and corrected. This could include discontinuation of antiplatelets where appropriate. Third, dose adjustments of direct anticoagulants could be reviewed such as the &#x201C;ABC&#x201D; criteria for apixaban or dose reductions for secondary venous thromboembolism prevention after 6 months. Fourth, this tool could be combined with electronic measures of International Normalized Ratio (INR) stability for those patients receiving warfarin therapy. While engaging the practice after a major bleed or intracranial bleed has occurred is not ideal, the use of this tool could have a major impact on practice change to prevent the next bleeding event. Fifth, these tools can provide a real-world, in-house assessment of new antithrombotic strategies. As the factor XI inhibitors are Food and Drug Administration (FDA)&#x2013;approved and incorporated into clinical practice, adverse outcomes, compared with traditional anticoagulants, can be assessed in real-time. Clearly, more work is needed to refine this and other tools to maximize utility. These types of tools may then enable incorporation into clinical practice for quality oversight.</p></sec><sec id="s4-2"><title>Comparison to Prior Work</title><p>Multiple recent studies have evaluated the ability to detect and categorize bleeding events using structured [<xref ref-type="bibr" rid="ref15">15</xref>,<xref ref-type="bibr" rid="ref20">20</xref>], unstructured [<xref ref-type="bibr" rid="ref21">21</xref>], or a combination of structured and unstructured data [<xref ref-type="bibr" rid="ref22">22</xref>,<xref ref-type="bibr" rid="ref23">23</xref>]. Coumau et al [<xref ref-type="bibr" rid="ref23">23</xref>] evaluated various approaches for detecting major bleeding as well as clinically relevant nonmajor bleeding. Their top approach was a combination of structured and unstructured data and yielded a sensitivity of 0.84, a precision of 0.51, and an <italic>F</italic><sub>1</sub>-score of 0.64. Gaspar et al [<xref ref-type="bibr" rid="ref21">21</xref>] performed document-level categorization of discharge summaries of older adults and compared an NLP approach with a rule-based, Dx Code approach. The NLP approach outperformed the rule-based, Dx Code approach in the ability to categorize a document as &#x201C;no bleeding,&#x201D; &#x201C;clinically significant bleeding,&#x201D; &#x201C;severe bleeding,&#x201D; and &#x201C;history of bleeding&#x201D; with an <italic>F</italic><sub>1</sub>-score of 0.80. An et al [<xref ref-type="bibr" rid="ref20">20</xref>] applied a previously published algorithm [<xref ref-type="bibr" rid="ref15">15</xref>] using structured data to detect major bleeding and clinically relevant nonmajor bleeding. They report a sensitivity of 0.91 and specificity of 0.92 for detecting major bleeding and a sensitivity of 0.52 and specificity of 0.92 for detecting clinically relevant nonmajor bleeding. They used a conditional sampling approach for the validation.</p><p>Older studies have used similar approaches to scrutinize clinical outcomes among large cohorts of anticoagulated patients. Cunningham et al [<xref ref-type="bibr" rid="ref9">9</xref>] used Dx Codes in addition to transfusion data to identify bleeding related to anticoagulation and achieved an 89% positive predictive value. Negative predictive value was not reported. Arnason et al [<xref ref-type="bibr" rid="ref10">10</xref>] reviewed patient charts with a &#x201C;bleeding&#x201D; Dx Code and determined if a major bleed occurred. Accuracy measures with 95% CIs were reported. Definite bleeding had a sensitivity of 93% (95% CI 90%&#x2010;96%) and specificity of 88% (95% CI 83%&#x2010;91%). Major bleeding had a sensitivity of 94% (95% CI 91%&#x2010;96%) and specificity of 83% (95% CI 78%&#x2010;87%). Tirschwell and Longstreth [<xref ref-type="bibr" rid="ref11">11</xref>] used Dx Codes to detect hemorrhagic stroke in administrative claims data, including primary diagnosis only, first 2 diagnoses, and all diagnoses. The best algorithm differed by stroke type, and for intracranial hemorrhage and subarachnoid hemorrhage, using only the primary diagnosis maximized agreement. Intracranial hemorrhage had sensitivity of 82% (95% CI 66%&#x2010;92%), specificity of 93% (95% CI 86%&#x2010;97%), and &#x03BA; 0.74 (0.58&#x2010;0.90), while subarachnoid hemorrhage had a sensitivity of 98% (95% CI 90%&#x2010;100%), specificity of 92% (95% CI 84%&#x2010;96%), and &#x03BA; 0.87 (95% CI 0.71&#x2010;1.00). While Dx Code approaches can be automated, they may not have sufficient accuracy metrics to be used in practice or within a learning health system. For example, a method with a sensitivity of 98% and a specificity of 92% applied to a population of 100,000 and assuming a 1% bleeding rate would result in 9780 false positives and 20 false negatives and a precision of 11%.</p><p>Others have assessed strategies that do not use Dx Codes for bleeding detection. Lee et al [<xref ref-type="bibr" rid="ref24">24</xref>] studied bleeding events related to clopidogrel usage. Ten different approaches, 3 rule-based, 3 score-based, and 4 machine learning&#x2013;based approaches were analyzed. While the highest precision was 97%, the sensitivity was low at 29%. A machine learning approach performed best, with a precision of 89%, a sensitivity of 97% (sensitivity in this study may not be accurate due to nonrandom sampling), and an <italic>F</italic><sub>1</sub>-score of 92%. Taggart et al [<xref ref-type="bibr" rid="ref25">25</xref>] searched for &#x201C;clinically relevant bleeding&#x201D; in the general adult intensive care unit population using the MIMIC-III (Medical Information Mart for Intensive Care) database. Using NLP results, they created rule-based and machine learning approaches for detecting bleeding. &#x201C;Clinically relevant bleeding&#x201D; was defined as &#x201C;as symptomatic bleeding that required treatment.&#x201D; The rule-based approach had a sensitivity of 91%, specificity of 85%, and precision of 63%. The machine learning&#x2013;based approach had a sensitivity of 94%, specificity of 72%, and precision of 49%. Shung et al [<xref ref-type="bibr" rid="ref26">26</xref>] assessed gastrointestinal bleeding in the ED. An early large language model (LLM) was the best approach (sensitivity of 93%, specificity of 63%, and precision of 84%) compared with either an NLP machine learning model and a discrete data model. More recent studies have focused on bleeding mention detection at the sentence or note level [<xref ref-type="bibr" rid="ref27">27</xref>,<xref ref-type="bibr" rid="ref28">28</xref>]. Further studies are needed to determine how sentence or note level detection would translate into patient level detection of bleeding events.</p><p>LLMs for information extraction present a potential future step beyond rule-based NLP that may have a superior ability to distinguish between bleeding events and no bleeding. Limitations of LLM approaches include potentially high cost, processing time and power requirements, and data privacy concerns. While recent advances suggest that LLMs and deep learning language models may be superior to rule-based NLP [<xref ref-type="bibr" rid="ref29">29</xref>-<xref ref-type="bibr" rid="ref31">31</xref>], 1 recent study found that rule-based NLP remained superior [<xref ref-type="bibr" rid="ref32">32</xref>]. One potential future direction of this work is to compare and/or combine the NLP&#x2229;Dx Code Any approach with an LLM approach.</p><p>Study strengths include the use of a rule-based NLP system to detect major bleeding for over 100,000 patients prescribed outpatient anticoagulation. This proof of concept demonstrates that large-scale detection of bleeding with NLP is feasible. Train, test, and validation cases were adjudicated by multiple reviewers to ensure accuracy. Two commonly used approaches were compared with a novel NLP approach that combines structured data of hemoglobin and transfusions to approximate the ISTH major bleed definition and scrutinizes both clinical and radiology notes for bleeding detection. An open-source NLP program that can be applied to large numbers of patients and notes was used at minimal expense.</p></sec><sec id="s4-3"><title>Limitations</title><p>The study has several limitations. First, only the incident bleeding event per patient was included. Yet, it is well known that patients can experience more than one bleeding event. Indeed, those patients with recurrent bleeding may be fundamentally different compared to those with an isolated event. By restricting the outcome to the initial bleed without further refinement, important clinical variables may be missed. Second, while considerable time was spent to thoroughly validate 1189 patients, this represents only 1% of the total population at risk. Whether this was an adequately representative sample is not known. Third, patients who suffered a major bleed were not necessarily on anticoagulation at the time. There may be fundamental differences between anticoagulation-related major bleeding and bleeding events occurring without anticoagulation. Fourth, in the manual adjudication, patients experiencing a bleeding complication related to surgery were excluded. These represent a unique and important cohort; however, uncontrollable variables related to surgical technique, surgical site, and extent are difficult to control for and unique to surgical populations. Surgical bleeding is a topic unto itself and not easily comprehensible when combined with nonsurgical bleeding. Fifth, the approach was not externally validated and relies on infrastructure that may be unique to Mayo Clinic. Sixth, the Cohen &#x03BA; metric on the validation sample was 0.66 (95% CI 0.38&#x2010;0.88), which is lower than desired. Adjudicating bleeding events is challenging, and discrepancies between reviewers were resolved by physician review or group consensus.</p></sec><sec id="s4-4"><title>Conclusions</title><p>A rule-based NLP pipeline processing tool was developed and validated for analysis of more than 100,000 patients over a period of 5 years. These kinds of tools are essential for developing quality improvement programs with the structured delivery of high-quality anticoagulation for large patient populations. While the &#x201C;NLP&#x2229;Dx Code&#x201D; in any position approach yielded excellent precision, the sensitivity of this approach needs to be improved. Future directions will combine this rule-based and Dx Code approach with new LLM-based approaches to information extraction.</p></sec></sec></body><back><ack><p>Che Ngufor and Shrinath Patel reviewed machine learning scripts and provided technical guidance. David Hodge reviewed the statistical plan.</p></ack><notes><sec><title>Generative AI Use</title><p>Generative AI was not used in the drafting of this manuscript.</p></sec><sec><title>Funding</title><p>The authors declared no financial support was received for this work.</p></sec><sec><title>Data Availability</title><p>The datasets generated or analyzed during this study are not publicly available due to the presence of protected health information (PHI) in clinical notes and radiology reports. Summary counts of natural language processing findings and/or code are available from the corresponding author upon reasonable request.</p></sec></notes><fn-group><fn fn-type="con"><p>Conceptualization: EWD, SS, RDM</p><p>Data curation: EWD (lead), JAA, SS, LB, MRH, KDN, JML, AEA, JGO, NDJ, RDM</p><p>Formal analysis: EWD</p><p>Investigation: EWD</p><p>Supervision: SS, RDM</p><p>Writing &#x2013; original draft: EWD</p><p>Writing &#x2013; review &#x0026; editing: EWD, JAA, SS, LB, MRH, KDN, JML, AEA, JGO, NDJ, RDM</p></fn><fn fn-type="conflict"><p>None declared.</p></fn></fn-group><glossary><title>Abbreviations</title><def-list><def-item><term id="abb1">CNS</term><def><p>central nervous system</p></def></def-item><def-item><term id="abb2">Dx Code</term><def><p>diagnosis code</p></def></def-item><def-item><term id="abb3">ED</term><def><p>emergency department</p></def></def-item><def-item><term id="abb4">EHR</term><def><p>electronic health record</p></def></def-item><def-item><term id="abb5">FDA</term><def><p>Food and Drug Administration</p></def></def-item><def-item><term id="abb6"><italic>ICD-10</italic></term><def><p><italic>International Statistical Classification of Diseases, Tenth Revision</italic></p></def></def-item><def-item><term id="abb7">INR</term><def><p>International Normalized Ratio</p></def></def-item><def-item><term id="abb8">ISTH</term><def><p>International Society on Thrombosis and Haemostasis</p></def></def-item><def-item><term id="abb9">LLM</term><def><p>large language model</p></def></def-item><def-item><term id="abb10">MIMIC-III</term><def><p>Medical Information Mart for Intensive Care</p></def></def-item><def-item><term id="abb11">NLP</term><def><p>natural language processing</p></def></def-item><def-item><term id="abb12">OHNLP</term><def><p>Open Health Natural Language Processing</p></def></def-item><def-item><term id="abb13">PRBC</term><def><p>packed red blood cells</p></def></def-item></def-list></glossary><ref-list><title>References</title><ref id="ref1"><label>1</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Yates</surname><given-names>NY</given-names> </name><name name-style="western"><surname>Hale</surname><given-names>SA</given-names> </name><name name-style="western"><surname>Clark</surname><given-names>NP</given-names> </name></person-group><article-title>The impact of clinical pharmacy services on direct oral anticoagulant medication selection and dosing in the ambulatory care setting</article-title><source>J Pharm Pract</source><year>2024</year><month>06</month><volume>37</volume><issue>3</issue><fpage>671</fpage><lpage>676</lpage><pub-id pub-id-type="doi">10.1177/08971900231166555</pub-id><pub-id pub-id-type="medline">36989436</pub-id></nlm-citation></ref><ref id="ref2"><label>2</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Gupta</surname><given-names>M</given-names> </name><name name-style="western"><surname>Gulati</surname><given-names>S</given-names> </name><name name-style="western"><surname>Nasir</surname><given-names>K</given-names> </name><name name-style="western"><surname>Sarraju</surname><given-names>A</given-names> </name></person-group><article-title>Aspirin use prevalence for cardiovascular disease prevention among U.S. adults from 2012 to 2021</article-title><source>Ann Intern Med</source><year>2024</year><month>08</month><volume>177</volume><issue>8</issue><fpage>1139</fpage><lpage>1141</lpage><pub-id pub-id-type="doi">10.7326/M24-0427</pub-id><pub-id pub-id-type="medline">38914002</pub-id></nlm-citation></ref><ref id="ref3"><label>3</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Geller</surname><given-names>AI</given-names> </name><name name-style="western"><surname>Shehab</surname><given-names>N</given-names> </name><name name-style="western"><surname>Lovegrove</surname><given-names>MC</given-names> </name><name name-style="western"><surname>Weidle</surname><given-names>NJ</given-names> </name><name name-style="western"><surname>Budnitz</surname><given-names>DS</given-names> </name></person-group><article-title>Bleeding related to oral anticoagulants: trends in US emergency department visits, 2016-2020</article-title><source>Thromb Res</source><year>2023</year><month>05</month><volume>225</volume><fpage>110</fpage><lpage>115</lpage><pub-id pub-id-type="doi">10.1016/j.thromres.2023.03.010</pub-id><pub-id pub-id-type="medline">37062120</pub-id></nlm-citation></ref><ref id="ref4"><label>4</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Siontis</surname><given-names>KC</given-names> </name><name name-style="western"><surname>Zhang</surname><given-names>X</given-names> </name><name name-style="western"><surname>Eckard</surname><given-names>A</given-names> </name><etal/></person-group><article-title>Outcomes associated with apixaban use in patients with end-stage kidney disease and atrial fibrillation in the United States</article-title><source>Circulation</source><year>2018</year><month>10</month><day>9</day><volume>138</volume><issue>15</issue><fpage>1519</fpage><lpage>1529</lpage><pub-id pub-id-type="doi">10.1161/CIRCULATIONAHA.118.035418</pub-id><pub-id pub-id-type="medline">29954737</pub-id></nlm-citation></ref><ref id="ref5"><label>5</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Reynolds</surname><given-names>MR</given-names> </name><name name-style="western"><surname>Gunnarsson</surname><given-names>CL</given-names> </name><name name-style="western"><surname>Hunter</surname><given-names>TD</given-names> </name><etal/></person-group><article-title>Health outcomes with catheter ablation or antiarrhythmic drug therapy in atrial fibrillation: results of a propensity-matched analysis</article-title><source>Circ Cardiovasc Qual Outcomes</source><year>2012</year><month>03</month><day>1</day><volume>5</volume><issue>2</issue><fpage>171</fpage><lpage>181</lpage><pub-id pub-id-type="doi">10.1161/CIRCOUTCOMES.111.963108</pub-id><pub-id pub-id-type="medline">22373904</pub-id></nlm-citation></ref><ref id="ref6"><label>6</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Jun</surname><given-names>M</given-names> </name><name name-style="western"><surname>James</surname><given-names>MT</given-names> </name><name name-style="western"><surname>Manns</surname><given-names>BJ</given-names> </name><etal/></person-group><article-title>The association between kidney function and major bleeding in older adults with atrial fibrillation starting warfarin treatment: population based observational study</article-title><source>BMJ</source><year>2015</year><month>02</month><day>3</day><volume>350</volume><fpage>h246</fpage><pub-id pub-id-type="doi">10.1136/bmj.h246</pub-id><pub-id pub-id-type="medline">25647223</pub-id></nlm-citation></ref><ref id="ref7"><label>7</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>White</surname><given-names>RH</given-names> </name><name name-style="western"><surname>Beyth</surname><given-names>RJ</given-names> </name><name name-style="western"><surname>Zhou</surname><given-names>H</given-names> </name><name name-style="western"><surname>Romano</surname><given-names>PS</given-names> </name></person-group><article-title>Major bleeding after hospitalization for deep-venous thrombosis</article-title><source>Am J Med</source><year>1999</year><month>11</month><volume>107</volume><issue>5</issue><fpage>414</fpage><lpage>424</lpage><pub-id pub-id-type="doi">10.1016/s0002-9343(99)00267-3</pub-id><pub-id pub-id-type="medline">10569295</pub-id></nlm-citation></ref><ref id="ref8"><label>8</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Ngufor</surname><given-names>C</given-names> </name><name name-style="western"><surname>Yao</surname><given-names>X</given-names> </name><name name-style="western"><surname>Inselman</surname><given-names>JW</given-names> </name><etal/></person-group><article-title>Identifying treatment heterogeneity in atrial fibrillation using a novel causal machine learning method</article-title><source>Am Heart J</source><year>2023</year><month>06</month><volume>260</volume><fpage>124</fpage><lpage>140</lpage><pub-id pub-id-type="doi">10.1016/j.ahj.2023.02.015</pub-id><pub-id pub-id-type="medline">36893934</pub-id></nlm-citation></ref><ref id="ref9"><label>9</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Cunningham</surname><given-names>A</given-names> </name><name name-style="western"><surname>Stein</surname><given-names>CM</given-names> </name><name name-style="western"><surname>Chung</surname><given-names>CP</given-names> </name><name name-style="western"><surname>Daugherty</surname><given-names>JR</given-names> </name><name name-style="western"><surname>Smalley</surname><given-names>WE</given-names> </name><name name-style="western"><surname>Ray</surname><given-names>WA</given-names> </name></person-group><article-title>An automated database case definition for serious bleeding related to oral anticoagulant use</article-title><source>Pharmacoepidemiol Drug Saf</source><year>2011</year><month>06</month><volume>20</volume><issue>6</issue><fpage>560</fpage><lpage>566</lpage><pub-id pub-id-type="doi">10.1002/pds.2109</pub-id><pub-id pub-id-type="medline">21387461</pub-id></nlm-citation></ref><ref id="ref10"><label>10</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Arnason</surname><given-names>T</given-names> </name><name name-style="western"><surname>Wells</surname><given-names>PS</given-names> </name><name name-style="western"><surname>van Walraven</surname><given-names>C</given-names> </name><name name-style="western"><surname>Forster</surname><given-names>AJ</given-names> </name></person-group><article-title>Accuracy of coding for possible warfarin complications in hospital discharge abstracts</article-title><source>Thromb Res</source><year>2006</year><volume>118</volume><issue>2</issue><fpage>253</fpage><lpage>262</lpage><pub-id pub-id-type="doi">10.1016/j.thromres.2005.06.015</pub-id><pub-id pub-id-type="medline">16081144</pub-id></nlm-citation></ref><ref id="ref11"><label>11</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Tirschwell</surname><given-names>DL</given-names> </name><name name-style="western"><surname>Longstreth</surname><given-names>WT</given-names> </name></person-group><article-title>Validating administrative data in stroke research</article-title><source>Stroke</source><year>2002</year><month>10</month><volume>33</volume><issue>10</issue><fpage>2465</fpage><lpage>2470</lpage><pub-id pub-id-type="doi">10.1161/01.str.0000032240.28636.bd</pub-id><pub-id pub-id-type="medline">12364739</pub-id></nlm-citation></ref><ref id="ref12"><label>12</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Schulman</surname><given-names>S</given-names> </name><name name-style="western"><surname>Kearon</surname><given-names>C</given-names> </name><collab>Subcommittee on Control of Anticoagulation of the Scientific and Standardization Committee of the International Society on Thrombosis and Haemostasis</collab></person-group><article-title>Definition of major bleeding in clinical investigations of antihemostatic medicinal products in non-surgical patients</article-title><source>J Thromb Haemost</source><year>2005</year><month>04</month><volume>3</volume><issue>4</issue><fpage>692</fpage><lpage>694</lpage><pub-id pub-id-type="doi">10.1111/j.1538-7836.2005.01204.x</pub-id><pub-id pub-id-type="medline">15842354</pub-id></nlm-citation></ref><ref id="ref13"><label>13</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Hauk</surname><given-names>L</given-names> </name></person-group><article-title>Bleeding with anticoagulants: ACC releases expert consensus decision pathway to guide management</article-title><source>Am Fam Physician</source><year>2018</year><month>07</month><day>1</day><volume>98</volume><issue>1</issue><fpage>57</fpage><lpage>58</lpage><pub-id pub-id-type="medline">30215949</pub-id></nlm-citation></ref><ref id="ref14"><label>14</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Tomaselli</surname><given-names>GF</given-names> </name><name name-style="western"><surname>Mahaffey</surname><given-names>KW</given-names> </name><name name-style="western"><surname>Cuker</surname><given-names>A</given-names> </name><etal/></person-group><article-title>2020 ACC expert consensus decision pathway on management of bleeding in patients on oral anticoagulants: a report of the American College of Cardiology Solution Set Oversight Committee</article-title><source>J Am Coll Cardiol</source><year>2020</year><month>08</month><day>4</day><volume>76</volume><issue>5</issue><fpage>594</fpage><lpage>622</lpage><pub-id pub-id-type="doi">10.1016/j.jacc.2020.04.053</pub-id><pub-id pub-id-type="medline">32680646</pub-id></nlm-citation></ref><ref id="ref15"><label>15</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Hartenstein</surname><given-names>A</given-names> </name><name name-style="western"><surname>Abdelgawwad</surname><given-names>K</given-names> </name><name name-style="western"><surname>Kleinjung</surname><given-names>F</given-names> </name><name name-style="western"><surname>Privitera</surname><given-names>S</given-names> </name><name name-style="western"><surname>Viethen</surname><given-names>T</given-names> </name><name name-style="western"><surname>Vaitsiakhovich</surname><given-names>T</given-names> </name></person-group><article-title>Identification of International Society on Thrombosis and Haemostasis major and clinically relevant non-major bleed events from electronic health records: a novel algorithm to enhance data utilisation from real-world sources</article-title><source>Int J Popul Data Sci</source><year>2023</year><volume>8</volume><issue>1</issue><fpage>2144</fpage><pub-id pub-id-type="doi">10.23889/ijpds.v8i1.2144</pub-id><pub-id pub-id-type="medline">38414540</pub-id></nlm-citation></ref><ref id="ref16"><label>16</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Liu</surname><given-names>H</given-names> </name><name name-style="western"><surname>Bielinski</surname><given-names>SJ</given-names> </name><name name-style="western"><surname>Sohn</surname><given-names>S</given-names> </name><etal/></person-group><article-title>An information extraction framework for cohort identification using electronic health records</article-title><source>AMIA Jt Summits Transl Sci Proc</source><year>2013</year><volume>2013</volume><fpage>149</fpage><lpage>153</lpage><pub-id pub-id-type="medline">24303255</pub-id></nlm-citation></ref><ref id="ref17"><label>17</label><nlm-citation citation-type="report"><person-group person-group-type="author"><name name-style="western"><surname>Settles</surname><given-names>B</given-names> </name></person-group><article-title>Active learning literature survey</article-title><year>2009</year><access-date>2025-03-10</access-date><publisher-name>University of Wisconsin&#x2013;Madison</publisher-name><comment><ext-link ext-link-type="uri" xlink:href="https://minds.wisc.edu/server/api/core/bitstreams/8a78cf83-0702-4157-aeff-3cacb62f9ad5/content">https://minds.wisc.edu/server/api/core/bitstreams/8a78cf83-0702-4157-aeff-3cacb62f9ad5/content</ext-link></comment></nlm-citation></ref><ref id="ref18"><label>18</label><nlm-citation citation-type="web"><article-title>proportion_ci</article-title><source>SciPy</source><access-date>2026-04-01</access-date><comment><ext-link ext-link-type="uri" xlink:href="https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats._result_classes.BinomTestResult.proportion_ci.html#r67c9937f472a-3">https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats._result_classes.BinomTestResult.proportion_ci.html#r67c9937f472a-3</ext-link></comment></nlm-citation></ref><ref id="ref19"><label>19</label><nlm-citation citation-type="other"><person-group person-group-type="author"><name name-style="western"><surname>Lam</surname><given-names>KFY</given-names> </name><name name-style="western"><surname>Gopal</surname><given-names>V</given-names> </name><name name-style="western"><surname>Qian</surname><given-names>J</given-names> </name></person-group><article-title>Confidence intervals for the F1 score: a comparison of four methods</article-title><source>arXiv</source><comment>Preprint posted online on  Sep 26, 2023</comment><pub-id pub-id-type="doi">10.48550/ARXIV.2309.14621</pub-id></nlm-citation></ref><ref id="ref20"><label>20</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>An</surname><given-names>J</given-names> </name><name name-style="western"><surname>Hartenstein</surname><given-names>A</given-names> </name><name name-style="western"><surname>Choi</surname><given-names>SK</given-names> </name><etal/></person-group><article-title>Validation of electronic health record-based International Society on Thrombosis and Haemostasis bleeding algorithms in a US health system</article-title><source>Res Pract Thromb Haemost</source><year>2026</year><month>04</month><volume>10</volume><issue>4</issue><fpage>106616</fpage><pub-id pub-id-type="doi">10.1016/j.rpth.2026.106616</pub-id><pub-id pub-id-type="medline">42232248</pub-id></nlm-citation></ref><ref id="ref21"><label>21</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Gaspar</surname><given-names>F</given-names> </name><name name-style="western"><surname>Zayene</surname><given-names>M</given-names> </name><name name-style="western"><surname>Coumau</surname><given-names>C</given-names> </name><etal/></person-group><article-title>Natural language processing and ICD-10 coding for detecting bleeding events in discharge summaries: comparative cross-sectional study</article-title><source>JMIR Med Inform</source><year>2025</year><month>08</month><day>29</day><volume>13</volume><fpage>e67837</fpage><pub-id pub-id-type="doi">10.2196/67837</pub-id><pub-id pub-id-type="medline">40882207</pub-id></nlm-citation></ref><ref id="ref22"><label>22</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Zheng</surname><given-names>NS</given-names> </name><name name-style="western"><surname>Keloth</surname><given-names>VK</given-names> </name><name name-style="western"><surname>You</surname><given-names>K</given-names> </name><etal/></person-group><article-title>Detection of gastrointestinal bleeding with large language models to aid quality improvement and appropriate reimbursement</article-title><source>Gastroenterology</source><year>2025</year><month>01</month><volume>168</volume><issue>1</issue><fpage>111</fpage><lpage>120</lpage><pub-id pub-id-type="doi">10.1053/j.gastro.2024.09.014</pub-id><pub-id pub-id-type="medline">39304088</pub-id></nlm-citation></ref><ref id="ref23"><label>23</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Coumau</surname><given-names>C</given-names> </name><name name-style="western"><surname>Gaspar</surname><given-names>F</given-names> </name><name name-style="western"><surname>Zayene</surname><given-names>M</given-names> </name><etal/></person-group><article-title>Detection of antithrombotic-related bleeding in older inpatients: multicenter retrospective study using structured and unstructured electronic health record data</article-title><source>J Med Internet Res</source><year>2026</year><month>01</month><day>29</day><volume>28</volume><fpage>e77809</fpage><pub-id pub-id-type="doi">10.2196/77809</pub-id><pub-id pub-id-type="medline">41610431</pub-id></nlm-citation></ref><ref id="ref24"><label>24</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Lee</surname><given-names>HJ</given-names> </name><name name-style="western"><surname>Jiang</surname><given-names>M</given-names> </name><name name-style="western"><surname>Wu</surname><given-names>Y</given-names> </name><etal/></person-group><article-title>A comparative study of different methods for automatic identification of clopidogrel-induced bleedings in electronic health records</article-title><source>AMIA Jt Summits Transl Sci Proc</source><year>2017</year><volume>2017</volume><issue>185&#x2013;192</issue><fpage>185</fpage><lpage>192</lpage><pub-id pub-id-type="medline">28815128</pub-id></nlm-citation></ref><ref id="ref25"><label>25</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Taggart</surname><given-names>M</given-names> </name><name name-style="western"><surname>Chapman</surname><given-names>WW</given-names> </name><name name-style="western"><surname>Steinberg</surname><given-names>BA</given-names> </name><etal/></person-group><article-title>Comparison of 2 natural language processing methods for identification of bleeding among critically ill patients</article-title><source>JAMA Netw Open</source><year>2018</year><month>10</month><day>5</day><volume>1</volume><issue>6</issue><fpage>e183451</fpage><pub-id pub-id-type="doi">10.1001/jamanetworkopen.2018.3451</pub-id><pub-id pub-id-type="medline">30646240</pub-id></nlm-citation></ref><ref id="ref26"><label>26</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Shung</surname><given-names>D</given-names> </name><name name-style="western"><surname>Tsay</surname><given-names>C</given-names> </name><name name-style="western"><surname>Laine</surname><given-names>L</given-names> </name><etal/></person-group><article-title>Early identification of patients with acute gastrointestinal bleeding using natural language processing and decision rules</article-title><source>J Gastroenterol Hepatol</source><year>2021</year><month>06</month><volume>36</volume><issue>6</issue><fpage>1590</fpage><lpage>1597</lpage><pub-id pub-id-type="doi">10.1111/jgh.15313</pub-id><pub-id pub-id-type="medline">33105045</pub-id></nlm-citation></ref><ref id="ref27"><label>27</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Pedersen</surname><given-names>JS</given-names> </name><name name-style="western"><surname>Laursen</surname><given-names>MS</given-names> </name><name name-style="western"><surname>Rajeeth Savarimuthu</surname><given-names>T</given-names> </name><etal/></person-group><article-title>Deep learning detects and visualizes bleeding events in electronic health records</article-title><source>Res Pract Thromb Haemost</source><year>2021</year><month>05</month><volume>5</volume><issue>4</issue><fpage>e12505</fpage><pub-id pub-id-type="doi">10.1002/rth2.12505</pub-id><pub-id pub-id-type="medline">34013150</pub-id></nlm-citation></ref><ref id="ref28"><label>28</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Li</surname><given-names>R</given-names> </name><name name-style="western"><surname>Hu</surname><given-names>B</given-names> </name><name name-style="western"><surname>Liu</surname><given-names>F</given-names> </name><etal/></person-group><article-title>Detection of bleeding events in electronic health record notes using convolutional neural network models enhanced with recurrent neural network autoencoders: deep learning approach</article-title><source>JMIR Med Inform</source><year>2019</year><month>02</month><day>8</day><volume>7</volume><issue>1</issue><fpage>e10788</fpage><pub-id pub-id-type="doi">10.2196/10788</pub-id><pub-id pub-id-type="medline">30735140</pub-id></nlm-citation></ref><ref id="ref29"><label>29</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Wang</surname><given-names>X</given-names> </name><name name-style="western"><surname>Huang</surname><given-names>L</given-names> </name><name name-style="western"><surname>Xu</surname><given-names>S</given-names> </name><name name-style="western"><surname>Lu</surname><given-names>K</given-names> </name></person-group><article-title>How does a generative large language model perform on domain-specific information extraction?&#x2500;A comparison between GPT-4 and a rule-based method on band gap extraction</article-title><source>J Chem Inf Model</source><year>2024</year><month>10</month><day>28</day><volume>64</volume><issue>20</issue><fpage>7895</fpage><lpage>7904</lpage><pub-id pub-id-type="doi">10.1021/acs.jcim.4c00882</pub-id><pub-id pub-id-type="medline">39375999</pub-id></nlm-citation></ref><ref id="ref30"><label>30</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Patel</surname><given-names>PV</given-names> </name><name name-style="western"><surname>Davis</surname><given-names>C</given-names> </name><name name-style="western"><surname>Ralbovsky</surname><given-names>A</given-names> </name><etal/></person-group><article-title>Large language models outperform traditional natural language processing methods in extracting patient-reported outcomes in inflammatory bowel disease</article-title><source>Gastro Hep Adv</source><year>2024</year><volume>4</volume><issue>2</issue><fpage>100563</fpage><pub-id pub-id-type="doi">10.1016/j.gastha.2024.10.003</pub-id><pub-id pub-id-type="medline">39877865</pub-id></nlm-citation></ref><ref id="ref31"><label>31</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Lybarger</surname><given-names>K</given-names> </name><name name-style="western"><surname>Yetisgen</surname><given-names>M</given-names> </name><name name-style="western"><surname>Uzuner</surname><given-names>&#x00D6;</given-names> </name></person-group><article-title>The 2022 n2c2/UW shared task on extracting social determinants of health</article-title><source>J Am Med Inform Assoc</source><year>2023</year><month>07</month><day>19</day><volume>30</volume><issue>8</issue><fpage>1367</fpage><lpage>1378</lpage><pub-id pub-id-type="doi">10.1093/jamia/ocad012</pub-id><pub-id pub-id-type="medline">36795066</pub-id></nlm-citation></ref><ref id="ref32"><label>32</label><nlm-citation citation-type="journal"><person-group person-group-type="author"><name name-style="western"><surname>Patra</surname><given-names>BG</given-names> </name><name name-style="western"><surname>Lepow</surname><given-names>LA</given-names> </name><name name-style="western"><surname>Kasi Reddy Jagadeesh Kumar</surname><given-names>P</given-names> </name><etal/></person-group><article-title>Extracting social support and social isolation information from clinical psychiatry notes: comparing a rule-based natural language processing system and a large language model</article-title><source>J Am Med Inform Assoc</source><year>2025</year><month>01</month><day>1</day><volume>32</volume><issue>1</issue><fpage>218</fpage><lpage>226</lpage><pub-id pub-id-type="doi">10.1093/jamia/ocae260</pub-id><pub-id pub-id-type="medline">39423850</pub-id></nlm-citation></ref></ref-list><app-group><supplementary-material id="app1"><label>Multimedia Appendix 1</label><p>Additional information regarding event adjudication, note search terms, MedTagger search terms, model training, diagnosis codes, and feature lists.</p><media xlink:href="cardio_v10i1e87424_app1.docx" xlink:title="DOCX File, 37 KB"/></supplementary-material></app-group></back></article>