From 1294bea29c38295166797eed9585474d06d546f6 Mon Sep 17 00:00:00 2001 From: solarsys Date: Fri, 9 Oct 2026 07:16:46 -0400 Subject: [PATCH 1/6] Add a nine-notebook tutorial series, from first model to contributing The Colab tutorials linked from the docs had drifted from the code: three pin pyhealth==2.0a4, one trains on 26 samples from the PhysioNet demo (validation ROC-AUC 1.0) and splits by sample, the text ones train Bio_ClinicalBERT on CPU for ~25 minutes per epoch without a baseline, and none covers patient-level splitting, calibration, interpretation or medical codes. examples/tutorials/series/ adds nine notebooks with a progression of complexity, each verified end to end on CPU against master with public or synthetic data: 00 quickstart, 01 datasets, 02 tasks and processors, 03 training and evaluation (PatientSplit, pos_weight, calibration, patient bootstrap CIs), 04 choosing a model (incl. XGBoost), 05 interpreting predictions (TreeSHAP, Integrated Gradients, deletion test), 06 medical codes, 07 clinical text, 08 contributing a dataset or task. The notebooks are generated from _source/tutorials/*.py by _source/build.py (stable cell ids; --run executes them). docs/tutorials.rst lists the series first, with Colab links that open from GitHub. Co-Authored-By: Claude Opus 5.5 --- docs/tutorials.rst | 40 +- examples/tutorials/series/00_quickstart.ipynb | 386 +++++++++++++++ examples/tutorials/series/01_datasets.ipynb | 454 +++++++++++++++++ .../series/02_tasks_and_processors.ipynb | 465 ++++++++++++++++++ .../series/03_training_and_evaluation.ipynb | 413 ++++++++++++++++ .../series/04_choosing_a_model.ipynb | 375 ++++++++++++++ .../series/05_interpreting_predictions.ipynb | 452 +++++++++++++++++ .../tutorials/series/06_medical_codes.ipynb | 343 +++++++++++++ .../tutorials/series/07_clinical_text.ipynb | 359 ++++++++++++++ .../tutorials/series/08_contributing.ipynb | 361 ++++++++++++++ examples/tutorials/series/README.md | 33 ++ examples/tutorials/series/_source/build.py | 76 +++ examples/tutorials/series/_source/nbkit.py | 93 ++++ .../series/_source/tutorials/__init__.py | 0 .../_source/tutorials/t00_quickstart.py | 214 ++++++++ .../series/_source/tutorials/t01_datasets.py | 275 +++++++++++ .../series/_source/tutorials/t02_tasks.py | 298 +++++++++++ .../series/_source/tutorials/t03_training.py | 246 +++++++++ .../series/_source/tutorials/t04_models.py | 244 +++++++++ .../series/_source/tutorials/t05_interpret.py | 286 +++++++++++ .../series/_source/tutorials/t06_medcode.py | 176 +++++++ .../series/_source/tutorials/t07_text.py | 204 ++++++++ .../_source/tutorials/t08_contributing.py | 256 ++++++++++ 23 files changed, 6048 insertions(+), 1 deletion(-) create mode 100644 examples/tutorials/series/00_quickstart.ipynb create mode 100644 examples/tutorials/series/01_datasets.ipynb create mode 100644 examples/tutorials/series/02_tasks_and_processors.ipynb create mode 100644 examples/tutorials/series/03_training_and_evaluation.ipynb create mode 100644 examples/tutorials/series/04_choosing_a_model.ipynb create mode 100644 examples/tutorials/series/05_interpreting_predictions.ipynb create mode 100644 examples/tutorials/series/06_medical_codes.ipynb create mode 100644 examples/tutorials/series/07_clinical_text.ipynb create mode 100644 examples/tutorials/series/08_contributing.ipynb create mode 100644 examples/tutorials/series/README.md create mode 100644 examples/tutorials/series/_source/build.py create mode 100644 examples/tutorials/series/_source/nbkit.py create mode 100644 examples/tutorials/series/_source/tutorials/__init__.py create mode 100644 examples/tutorials/series/_source/tutorials/t00_quickstart.py create mode 100644 examples/tutorials/series/_source/tutorials/t01_datasets.py create mode 100644 examples/tutorials/series/_source/tutorials/t02_tasks.py create mode 100644 examples/tutorials/series/_source/tutorials/t03_training.py create mode 100644 examples/tutorials/series/_source/tutorials/t04_models.py create mode 100644 examples/tutorials/series/_source/tutorials/t05_interpret.py create mode 100644 examples/tutorials/series/_source/tutorials/t06_medcode.py create mode 100644 examples/tutorials/series/_source/tutorials/t07_text.py create mode 100644 examples/tutorials/series/_source/tutorials/t08_contributing.py diff --git a/docs/tutorials.rst b/docs/tutorials.rst index 9193c86c0..fa110a726 100644 --- a/docs/tutorials.rst +++ b/docs/tutorials.rst @@ -1,7 +1,45 @@ Tutorials ======================== - We provide the following tutorials to help users get started with our pyhealth. Please bear with us as we update the documentation on how to use pyhealth 2.0. +Tutorial series +--------------- + +A step-by-step series, from a first model to contributing to PyHealth. Each +notebook opens in Google Colab, runs on a free CPU runtime, and uses public or +synthetic data, so no credentials are needed. The sources are in +`examples/tutorials/series `_. + +.. list-table:: + :widths: 40 60 + :header-rows: 1 + + * - Notebook + - What you will learn + * - `00. Quickstart: your first clinical prediction model `_ + - The five steps every project follows: dataset, task, split, model, evaluation. + * - `01. Datasets: patients, events and your own data `_ + - Exploring patients and events, YAML configs, caching, and loading your own CSV files. + * - `02. Tasks and processors `_ + - Writing your own task and what each processor turns raw values into. + * - `03. Training and evaluating models properly `_ + - Leakage-free patient splits, early stopping, class imbalance, calibration and bootstrap confidence intervals. + * - `04. Choosing a model `_ + - Logistic regression to Transformer, and an XGBoost baseline, compared on one split. + * - `05. Interpreting predictions `_ + - Exact TreeSHAP, Integrated Gradients, and checking that explanations are faithful. + * - `06. Medical codes `_ + - Code lookups, ICD-9/ICD-10 translation, CCS and ATC groupers, and code mapping inside tasks. + * - `07. Clinical text `_ + - Fine-tuning language models for specialty classification and ICD coding, with a TF-IDF baseline. + * - `08. Contributing a dataset or task `_ + - Dataset classes, file-based data, tests and the pull request checklist. + +Earlier tutorials +----------------- + +The notebooks below were written for earlier PyHealth versions; prefer the +series above. + `Tutorial 0: Introduction to pyhealth.data `_ `[Video] `_ diff --git a/examples/tutorials/series/00_quickstart.ipynb b/examples/tutorials/series/00_quickstart.ipynb new file mode 100644 index 000000000..debd35127 --- /dev/null +++ b/examples/tutorials/series/00_quickstart.ipynb @@ -0,0 +1,386 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-00", + "metadata": {}, + "source": [ + "# PyHealth Tutorial 00: Quickstart: your first clinical prediction model\n", + "\n", + "**What you will learn**\n", + "- The five steps every PyHealth project follows: dataset, task, split, model, evaluation\n", + "- How to predict in-hospital mortality from diagnosis, procedure and drug codes\n", + "- How to read the results, and why PR-AUC matters for rare outcomes\n", + "\n", + "**Time:** about 15 minutes on a free Colab CPU runtime. \n", + "**Before this:** nothing. This is the place to start.\n", + "\n", + "All data used here is synthetic or public, so every cell runs without credentials." + ] + }, + { + "cell_type": "markdown", + "id": "cell-01", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub. Once 2.1 is on PyPI this becomes `pip install \"pyhealth>=2.1\"`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-02", + "metadata": { + "tags": [ + "install" + ] + }, + "outputs": [], + "source": [ + "%pip install -q \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"" + ] + }, + { + "cell_type": "markdown", + "id": "cell-03", + "metadata": {}, + "source": [ + "## The PyHealth pipeline in one picture\n", + "\n", + "Every PyHealth project has the same five steps. Each one is a single\n", + "object, so you can swap a piece (another dataset, task or model)\n", + "without touching the rest:\n", + "\n", + "| Step | PyHealth object | What it does |\n", + "|---|---|---|\n", + "| 1. Load data | `pyhealth.datasets` (e.g. `MIMIC3Dataset`) | Reads raw tables into patients, each with a timeline of events |\n", + "| 2. Define the task | `pyhealth.tasks` (e.g. `MortalityPredictionMIMIC3`) | Turns each patient into labelled samples (features + label) |\n", + "| 3. Split | `split_by_patient` | Keeps each patient in only one of train / validation / test |\n", + "| 4. Train | `pyhealth.models` + `Trainer` | Fits a model on the training samples |\n", + "| 5. Evaluate | `Trainer.evaluate`, `pyhealth.metrics` | Scores the model on patients it has never seen |\n", + "\n", + "We will run all five on **synthetic MIMIC-III**: a public, generated\n", + "copy of the MIMIC-III intensive-care database with the same tables and\n", + "columns, but no real patients. Results on synthetic data say nothing\n", + "clinical; the point is to learn the workflow, which is identical on the\n", + "real database." + ] + }, + { + "cell_type": "markdown", + "id": "cell-04", + "metadata": {}, + "source": [ + "## Step 1. Load the dataset\n", + "\n", + "`MIMIC3Dataset` reads the raw MIMIC-III tables you ask for and links\n", + "every row to its patient. `root` can be a local folder or a URL; here\n", + "it points at the public synthetic copy.\n", + "\n", + "- `tables`: which tables to load. Patients and admissions are always\n", + " loaded; we add diagnoses, procedures and prescriptions.\n", + "- `cache_dir`: where PyHealth keeps its processed copy so the next run\n", + " is fast. With real patient data, put this inside storage you control.\n", + "\n", + "The first run downloads and processes the tables, which takes a couple\n", + "of minutes. PyHealth logs each step as it goes; the long log is normal." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-05", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.datasets import MIMIC3Dataset\n", + "\n", + "dataset = MIMIC3Dataset(\n", + " root=\"https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III\",\n", + " tables=[\"diagnoses_icd\", \"procedures_icd\", \"prescriptions\"],\n", + " cache_dir=\"pyhealth_cache\",\n", + ")\n", + "dataset.stats()" + ] + }, + { + "cell_type": "markdown", + "id": "cell-06", + "metadata": {}, + "source": [ + "A dataset is a collection of **patients**, and each patient is a\n", + "timeline of **events** (an admission, a diagnosis code, a drug order).\n", + "Let's look at one patient:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-07", + "metadata": {}, + "outputs": [], + "source": [ + "patient = dataset.get_patient(dataset.unique_patient_ids[0])\n", + "print(\"patient:\", patient.patient_id)\n", + "for event in patient.get_events()[:5]:\n", + " print(event)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-08", + "metadata": {}, + "source": [ + "Tutorial 01 explores datasets in depth. For now, the key idea: the\n", + "dataset holds *all* the data, and it does not yet know what you want to\n", + "predict. That is the task's job.\n", + "\n", + "## Step 2. Define the task\n", + "\n", + "A **task** says what one training example looks like. PyHealth ships\n", + "ready-made tasks for common problems. `MortalityPredictionMIMIC3`\n", + "creates one sample per hospital admission:\n", + "\n", + "- **Inputs**: the diagnosis codes (`conditions`), procedure codes\n", + " (`procedures`) and drug codes (`drugs`) recorded in that admission.\n", + "- **Label** (`mortality`): 1 if the patient died in hospital during\n", + " their *next* admission, else 0. Patients with a single admission\n", + " have no \"next\" admission, so they produce no samples.\n", + "\n", + "`set_task` runs the task over every patient and returns a\n", + "`SampleDataset`, ready for a model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-09", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.tasks import MortalityPredictionMIMIC3\n", + "\n", + "task = MortalityPredictionMIMIC3()\n", + "print(\"inputs:\", task.input_schema)\n", + "print(\"label: \", task.output_schema)\n", + "\n", + "samples = dataset.set_task(task)\n", + "print(f\"{len(samples)} samples\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-10", + "metadata": {}, + "source": [ + "The schemas tell PyHealth how to turn each field into numbers: a\n", + "`\"sequence\"` is a list of codes, mapped to integer ids with a\n", + "vocabulary learned from the data; `\"binary\"` is a 0/1 label.\n", + "\n", + "Here is one sample. The codes are already integer ids, and the label is\n", + "a tensor:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-11", + "metadata": {}, + "outputs": [], + "source": [ + "sample = samples[0]\n", + "for key, value in sample.items():\n", + " print(f\"{key:12s} {value}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-12", + "metadata": {}, + "source": [ + "Before training, check how common the outcome is. A model that always\n", + "says \"survives\" would be right most of the time, so accuracy alone is\n", + "misleading for rare outcomes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-13", + "metadata": {}, + "outputs": [], + "source": [ + "labels = [int(samples[i][\"mortality\"]) for i in range(len(samples))]\n", + "print(f\"{sum(labels)} of {len(labels)} samples are positive \"\n", + " f\"({sum(labels) / len(labels):.1%})\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-14", + "metadata": {}, + "source": [ + "## Step 3. Split by patient\n", + "\n", + "One patient can contribute several samples (one per admission). If the\n", + "same patient appears in both training and test data, the model can\n", + "recognise the patient instead of learning anything general, and the\n", + "test score looks better than it really is. `split_by_patient` puts\n", + "every patient in exactly one part: 70% train, 10% validation (to pick\n", + "the best epoch), 20% test (touched only once, at the end)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-15", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.datasets import get_dataloader, split_by_patient\n", + "\n", + "train_ds, val_ds, test_ds = split_by_patient(samples, [0.7, 0.1, 0.2], seed=42)\n", + "print(f\"train {len(train_ds)} val {len(val_ds)} test {len(test_ds)}\")\n", + "\n", + "train_loader = get_dataloader(train_ds, batch_size=64, shuffle=True)\n", + "val_loader = get_dataloader(val_ds, batch_size=64, shuffle=False)\n", + "test_loader = get_dataloader(test_ds, batch_size=64, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-16", + "metadata": {}, + "source": [ + "## Step 4. Build and train a model\n", + "\n", + "PyHealth models read the schemas from the dataset, so you do not\n", + "describe the inputs again. `RNN` embeds each code, runs a recurrent\n", + "network over each field, and combines them into one prediction.\n", + "\n", + "The `Trainer` handles the training loop: batches, optimizer, evaluation\n", + "on the validation set after each epoch, and keeping the best epoch by\n", + "the metric you `monitor`. We monitor **PR-AUC** (area under the\n", + "precision-recall curve), which focuses on how well the rare positive\n", + "cases are found." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-17", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.models import RNN\n", + "from pyhealth.trainer import Trainer\n", + "\n", + "model = RNN(dataset=samples)\n", + "trainer = Trainer(model=model, metrics=[\"pr_auc\", \"roc_auc\"])\n", + "trainer.train(\n", + " train_dataloader=train_loader,\n", + " val_dataloader=val_loader,\n", + " epochs=10,\n", + " monitor=\"pr_auc\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-18", + "metadata": {}, + "source": [ + "## Step 5. Evaluate on the test patients\n", + "\n", + "The trainer has reloaded the best epoch, so we score it once on the\n", + "held-out test patients.\n", + "\n", + "How to read the numbers:\n", + "- **ROC-AUC**: the chance that a random positive case is ranked above a\n", + " random negative one. 0.5 is a coin flip, 1.0 is perfect.\n", + "- **PR-AUC**: precision averaged over recall levels. Its no-skill\n", + " baseline is the positive rate printed above, not 0.5, so compare\n", + " against that." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-19", + "metadata": {}, + "outputs": [], + "source": [ + "scores = trainer.evaluate(test_loader)\n", + "prevalence = sum(int(test_ds[i][\"mortality\"]) for i in range(len(test_ds))) / len(test_ds)\n", + "print(f\"test ROC-AUC {scores['roc_auc']:.3f}\")\n", + "print(f\"test PR-AUC {scores['pr_auc']:.3f} (no-skill baseline {prevalence:.3f})\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-20", + "metadata": {}, + "source": [ + "On synthetic data the scores are close to chance: the generated\n", + "records have little real signal. On real MIMIC-III the same code learns\n", + "meaningful patterns.\n", + "\n", + "## What you did\n", + "\n", + "```python\n", + "dataset = MIMIC3Dataset(root=..., tables=[...]) # 1. data\n", + "samples = dataset.set_task(MortalityPredictionMIMIC3()) # 2. task\n", + "train, val, test = split_by_patient(samples, [0.7, 0.1, 0.2]) # 3. split\n", + "trainer = Trainer(model=RNN(dataset=samples)) # 4. model\n", + "trainer.train(...); trainer.evaluate(test_loader) # 5. evaluate\n", + "```\n", + "\n", + "To try a variation, change one line: another task\n", + "(`ReadmissionPredictionMIMIC3`), another model (`Transformer`,\n", + "`RETAIN`), or your own data (Tutorial 01)." + ] + }, + { + "cell_type": "markdown", + "id": "cell-21", + "metadata": {}, + "source": [ + "## Where to go next\n", + "\n", + "**Next:** Tutorial 01, *Datasets: patients, events and your own data*.\n", + "\n", + "The whole series:\n", + "\n", + "00. Quickstart: your first clinical prediction model\n", + "01. Datasets: patients, events and your own data\n", + "02. Tasks and processors: from patients to model-ready samples\n", + "03. Training and evaluating models properly\n", + "04. Choosing a model: from logistic regression to XGBoost\n", + "05. Interpreting predictions\n", + "06. Medical codes: lookups, mappings and groupers\n", + "07. Clinical text: classification and medical coding\n", + "08. Contributing a dataset or task to PyHealth\n", + "\n", + "Questions or problems? Open an issue at https://github.com/sunlabuiuc/PyHealth/issues. If PyHealth is useful to you, a star on GitHub helps the project." + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/tutorials/series/01_datasets.ipynb b/examples/tutorials/series/01_datasets.ipynb new file mode 100644 index 000000000..b9e1fce5d --- /dev/null +++ b/examples/tutorials/series/01_datasets.ipynb @@ -0,0 +1,454 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-00", + "metadata": {}, + "source": [ + "# PyHealth Tutorial 01: Datasets: patients, events and your own data\n", + "\n", + "**What you will learn**\n", + "- The difference between a dataset and a task, and why PyHealth keeps them apart\n", + "- How to explore patients and their events, filter by type, time or admission\n", + "- What the YAML config does, and how caching works\n", + "- How to load your own CSV files with a short config, no new class needed\n", + "\n", + "**Time:** about 20 minutes on a free Colab CPU runtime. \n", + "**Before this:** Tutorial 00 (Quickstart) is helpful but not required.\n", + "\n", + "All data used here is synthetic or public, so every cell runs without credentials." + ] + }, + { + "cell_type": "markdown", + "id": "cell-01", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub. Once 2.1 is on PyPI this becomes `pip install \"pyhealth>=2.1\"`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-02", + "metadata": { + "tags": [ + "install" + ] + }, + "outputs": [], + "source": [ + "%pip install -q \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"" + ] + }, + { + "cell_type": "markdown", + "id": "cell-03", + "metadata": {}, + "source": [ + "## A dataset is not a task\n", + "\n", + "![Datasets feed many tasks](https://drive.google.com/uc?export=view&id=1hHJcavXqisH9JEMqEVtE4TqEg_E489l5)\n", + "\n", + "A **dataset** in PyHealth is a pool of raw patient data: who the\n", + "patients are and everything that happened to them, in time order. It\n", + "does not know what you want to predict.\n", + "\n", + "A **task** (Tutorial 02) reads that pool and builds labelled examples\n", + "for one question. The same MIMIC-III dataset serves mortality\n", + "prediction, readmission prediction, length of stay, drug\n", + "recommendation and medical coding, each a different task.\n", + "\n", + "Keeping them apart means you load and clean the data once, then ask\n", + "many questions of it. When you contribute to PyHealth, a new data\n", + "*source* belongs in `pyhealth.datasets`; a new *question* about an\n", + "existing source (say, a new MIMIC-III outcome) belongs in\n", + "`pyhealth.tasks`." + ] + }, + { + "cell_type": "markdown", + "id": "cell-04", + "metadata": {}, + "source": [ + "## Loading a built-in dataset\n", + "\n", + "PyHealth has loaders for many sources, including `MIMIC3Dataset`,\n", + "`MIMIC4Dataset`, `eICUDataset`, `OMOPDataset` (any OMOP-CDM database),\n", + "and imaging, signal and text datasets. They all share the same\n", + "arguments:\n", + "\n", + "| Argument | Meaning |\n", + "|---|---|\n", + "| `root` | Folder or URL with the raw files |\n", + "| `tables` | Which tables to load (core tables such as patients and admissions are always added) |\n", + "| `cache_dir` | Where the processed copy is stored; reused on the next run |\n", + "| `dev` | `True` keeps only 1,000 patients, for quick experiments |\n", + "\n", + "We use the public synthetic MIMIC-III, with four tables. If you ran\n", + "Tutorial 00 in the same runtime, this loads from the cache in seconds.\n", + "PyHealth logs each step as it goes; the long log is normal." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-05", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.datasets import MIMIC3Dataset\n", + "\n", + "dataset = MIMIC3Dataset(\n", + " root=\"https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III\",\n", + " tables=[\"diagnoses_icd\", \"procedures_icd\", \"prescriptions\", \"noteevents\"],\n", + " cache_dir=\"pyhealth_cache\",\n", + ")\n", + "dataset.stats()" + ] + }, + { + "cell_type": "markdown", + "id": "cell-06", + "metadata": {}, + "source": [ + "## Patients and events\n", + "\n", + "Everything in a dataset is an **event**: one row of one table, tied to\n", + "a patient and (usually) a time. An event has:\n", + "\n", + "- `event_type`: the table it came from (`\"admissions\"`, `\"diagnoses_icd\"`, ...)\n", + "- `timestamp`: when it happened (`None` for timeless rows such as demographics)\n", + "- attributes: the table columns listed in the config, readable as\n", + " `event.icd9_code` or `event[\"icd9_code\"]`\n", + "\n", + "A **patient** is that patient's events in time order." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-07", + "metadata": {}, + "outputs": [], + "source": [ + "patient_ids = dataset.unique_patient_ids\n", + "patient = dataset.get_patient(patient_ids[0])\n", + "\n", + "events = patient.get_events()\n", + "print(f\"patient {patient.patient_id} has {len(events)} events\")\n", + "for event in events[:3]:\n", + " print(event.event_type, event.timestamp, dict(event.attr_dict))" + ] + }, + { + "cell_type": "markdown", + "id": "cell-08", + "metadata": {}, + "source": [ + "### Picking out the events you need\n", + "\n", + "`get_events` takes three kinds of filters, which you can combine:\n", + "\n", + "- `event_type=\"diagnoses_icd\"`: events from one table\n", + "- `start=` / `end=`: events within a time window\n", + "- `filters=[(\"hadm_id\", \"==\", \"...\")]`: events whose attribute matches,\n", + " for example everything recorded during one hospital admission" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-09", + "metadata": {}, + "outputs": [], + "source": [ + "# every diagnosis code this patient ever received\n", + "diagnoses = patient.get_events(event_type=\"diagnoses_icd\")\n", + "print(\"diagnosis codes:\", [e.icd9_code for e in diagnoses][:10])\n", + "\n", + "# the patient's admissions, in time order\n", + "admissions = patient.get_events(event_type=\"admissions\")\n", + "for a in admissions:\n", + " print(f\"admission {a.hadm_id}: {a.timestamp} -> {a.dischtime}, died in hospital: {a.hospital_expire_flag}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-10", + "metadata": {}, + "outputs": [], + "source": [ + "# what happened during the first admission only\n", + "first = admissions[0]\n", + "in_first = patient.get_events(event_type=\"prescriptions\", filters=[(\"hadm_id\", \"==\", first.hadm_id)])\n", + "print(f\"{len(in_first)} drug orders in admission {first.hadm_id}:\",\n", + " sorted({e.drug for e in in_first})[:8])\n", + "\n", + "# and everything from that admission's start onwards\n", + "later = patient.get_events(start=first.timestamp)\n", + "print(f\"{len(later)} events from {first.timestamp} on\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-11", + "metadata": {}, + "source": [ + "For analysis, `return_df=True` gives the events as a\n", + "[polars](https://pola.rs) DataFrame instead of a list. Attribute\n", + "columns are named `/`; the frame has a column for\n", + "every attribute of every loaded table, so select the ones you need:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-12", + "metadata": {}, + "outputs": [], + "source": [ + "df = patient.get_events(event_type=\"diagnoses_icd\", return_df=True)\n", + "df.select([\"timestamp\", \"diagnoses_icd/hadm_id\", \"diagnoses_icd/icd9_code\", \"diagnoses_icd/seq_num\"])" + ] + }, + { + "cell_type": "markdown", + "id": "cell-13", + "metadata": {}, + "source": [ + "Looping over many patients is the same idea. Here we count how many\n", + "patients have more than one admission, which is what the mortality task\n", + "in Tutorial 00 needs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-14", + "metadata": {}, + "outputs": [], + "source": [ + "from itertools import islice\n", + "\n", + "multi = 0\n", + "checked = 0\n", + "for pid in islice(patient_ids, 2000):\n", + " checked += 1\n", + " if len(dataset.get_patient(pid).get_events(event_type=\"admissions\")) > 1:\n", + " multi += 1\n", + "print(f\"{multi} of the first {checked} patients have 2+ admissions\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-15", + "metadata": {}, + "source": [ + "## Under the hood: the YAML config\n", + "\n", + "How does PyHealth know which column is the patient id, which is the\n", + "time, and which columns to keep? Each dataset has a YAML config. Here\n", + "is part of the one used above, read straight from the installed\n", + "package:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-16", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import pyhealth.datasets\n", + "\n", + "config = Path(pyhealth.datasets.__file__).parent / \"configs\" / \"mimic3.yaml\"\n", + "text = config.read_text()\n", + "start = text.index(\" diagnoses_icd:\")\n", + "print(text[:text.index(\" admissions:\")])\n", + "print(text[start:text.index(\"\\n\\n\", start)])" + ] + }, + { + "cell_type": "markdown", + "id": "cell-17", + "metadata": {}, + "source": [ + "Every table entry has:\n", + "\n", + "| Field | Meaning |\n", + "|---|---|\n", + "| `file_path` | The file under `root` |\n", + "| `patient_id` | The column that identifies the patient |\n", + "| `timestamp` | The column with the event time (`null` for timeless tables) |\n", + "| `attributes` | The columns to keep; they become event attributes |\n", + "| `join` (optional) | Columns to pull in from another file. `diagnoses_icd` has no time of its own, so it borrows `dischtime` from `ADMISSIONS` |\n", + "\n", + "## Caching\n", + "\n", + "Reading and joining the raw tables is the slow part, so PyHealth\n", + "stores the result under `cache_dir` and reuses it. The cache entry is\n", + "keyed on the dataset's settings and on the config and source files:\n", + "change a table or edit the config, and you get a fresh entry instead\n", + "of stale data. Task outputs (Tutorial 02) are cached the same way.\n", + "\n", + "Two practical notes:\n", + "- Without `cache_dir`, PyHealth uses a default folder in your home\n", + " directory and prints a warning saying where. With real patient\n", + " data, always pass a `cache_dir` inside storage you control.\n", + "- To make that mandatory (for example on a shared server), set the\n", + " environment variable `PYHEALTH_REQUIRE_CACHE_DIR=1`; loading a\n", + " dataset without `cache_dir` then raises an error." + ] + }, + { + "cell_type": "markdown", + "id": "cell-18", + "metadata": {}, + "source": [ + "## Your own data: CSV files plus a config\n", + "\n", + "You do not need to write a Python class to use your own data. Put each\n", + "table in a CSV file, describe them in a YAML config, and load them\n", + "with `BaseDataset`. Here we create a tiny clinic dataset with a\n", + "demographics table and a visits table." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-19", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "root = Path(\"my_clinic\")\n", + "root.mkdir(exist_ok=True)\n", + "\n", + "(root / \"patients.csv\").write_text(\n", + " \"patient_id,birth_year,sex\\n\"\n", + " \"p1,1956,F\\n\"\n", + " \"p2,1971,M\\n\"\n", + " \"p3,1990,F\\n\"\n", + ")\n", + "(root / \"visits.csv\").write_text(\n", + " \"patient_id,visit_time,diagnosis,systolic_bp\\n\"\n", + " \"p1,2023-01-05 09:30,I10,152\\n\"\n", + " \"p1,2023-06-12 14:00,E11.9,148\\n\"\n", + " \"p1,2024-02-01 10:15,I10,139\\n\"\n", + " \"p2,2023-03-20 08:45,J45.909,121\\n\"\n", + " \"p3,2023-11-02 16:20,I10,135\\n\"\n", + " \"p3,2024-04-18 11:05,N18.3,141\\n\"\n", + ")\n", + "(root / \"my_clinic.yaml\").write_text('''\n", + "version: \"1.0\"\n", + "tables:\n", + " patients:\n", + " file_path: \"patients.csv\"\n", + " patient_id: \"patient_id\"\n", + " timestamp: null\n", + " attributes: [\"birth_year\", \"sex\"]\n", + " visits:\n", + " file_path: \"visits.csv\"\n", + " patient_id: \"patient_id\"\n", + " timestamp: \"visit_time\"\n", + " timestamp_format: \"%Y-%m-%d %H:%M\"\n", + " attributes: [\"diagnosis\", \"systolic_bp\"]\n", + "''')\n", + "print(sorted(p.name for p in root.iterdir()))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-20", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.datasets import BaseDataset\n", + "\n", + "clinic = BaseDataset(\n", + " root=str(root),\n", + " tables=[\"patients\", \"visits\"],\n", + " dataset_name=\"my_clinic\",\n", + " config_path=str(root / \"my_clinic.yaml\"),\n", + " cache_dir=\"pyhealth_cache\",\n", + ")\n", + "clinic.stats()\n", + "\n", + "for event in clinic.get_patient(\"p1\").get_events(event_type=\"visits\"):\n", + " print(event.timestamp, event.diagnosis, event.systolic_bp)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-21", + "metadata": {}, + "source": [ + "That is all it takes: your data now works with every PyHealth task,\n", + "model and evaluation tool, exactly like MIMIC-III. Values are read as\n", + "text, so convert them in your task (for example\n", + "`float(event.systolic_bp)`).\n", + "\n", + "If you later want to share the loader with others, wrap it in a small\n", + "`BaseDataset` subclass and contribute it; Tutorial 08 walks through\n", + "that, using PyHealth's own MIMIC-III and COVID-19 X-ray loaders as\n", + "examples.\n", + "\n", + "## Summary\n", + "\n", + "- A dataset holds patients; a patient is a time-ordered list of events.\n", + "- `get_events(event_type=..., start=..., end=..., filters=...)` selects\n", + " what you need; `return_df=True` gives a DataFrame.\n", + "- A YAML config maps raw files to events; `cache_dir` keeps the\n", + " processed copy.\n", + "- Your own CSVs load with `BaseDataset` and a config." + ] + }, + { + "cell_type": "markdown", + "id": "cell-22", + "metadata": {}, + "source": [ + "## Where to go next\n", + "\n", + "**Next:** Tutorial 02, *Tasks and processors: from patients to model-ready samples*.\n", + "\n", + "The whole series:\n", + "\n", + "00. Quickstart: your first clinical prediction model\n", + "01. Datasets: patients, events and your own data\n", + "02. Tasks and processors: from patients to model-ready samples\n", + "03. Training and evaluating models properly\n", + "04. Choosing a model: from logistic regression to XGBoost\n", + "05. Interpreting predictions\n", + "06. Medical codes: lookups, mappings and groupers\n", + "07. Clinical text: classification and medical coding\n", + "08. Contributing a dataset or task to PyHealth\n", + "\n", + "Questions or problems? Open an issue at https://github.com/sunlabuiuc/PyHealth/issues. If PyHealth is useful to you, a star on GitHub helps the project." + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/tutorials/series/02_tasks_and_processors.ipynb b/examples/tutorials/series/02_tasks_and_processors.ipynb new file mode 100644 index 000000000..7166112bc --- /dev/null +++ b/examples/tutorials/series/02_tasks_and_processors.ipynb @@ -0,0 +1,465 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-00", + "metadata": {}, + "source": [ + "# PyHealth Tutorial 02: Tasks and processors: from patients to model-ready samples\n", + "\n", + "**What you will learn**\n", + "- What a task is: a function from one patient to a list of samples\n", + "- How input and output schemas pick a processor for each field\n", + "- What each common processor produces (sequence, nested sequence, multi-hot, tensor, labels)\n", + "- How to write your own task, filter patients early, and use it with any model\n", + "\n", + "**Time:** about 25 minutes on a free Colab CPU runtime. \n", + "**Before this:** Tutorial 01 (Datasets).\n", + "\n", + "All data used here is synthetic or public, so every cell runs without credentials." + ] + }, + { + "cell_type": "markdown", + "id": "cell-01", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub. Once 2.1 is on PyPI this becomes `pip install \"pyhealth>=2.1\"`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-02", + "metadata": { + "tags": [ + "install" + ] + }, + "outputs": [], + "source": [ + "%pip install -q \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"" + ] + }, + { + "cell_type": "markdown", + "id": "cell-03", + "metadata": {}, + "source": [ + "## Where tasks fit\n", + "\n", + "![Tasks build on datasets](https://drive.google.com/uc?export=view&id=1hHJcavXqisH9JEMqEVtE4TqEg_E489l5)\n", + "\n", + "A dataset holds every patient's events. A **task** answers one\n", + "question about them: for each patient it returns zero or more\n", + "**samples**, each a dictionary of input features and a label.\n", + "\n", + "Every task is a small class with three parts:\n", + "\n", + "```python\n", + "class MyTask(BaseTask):\n", + " task_name = \"my_task\" # used in cache names\n", + " input_schema = {\"conditions\": \"sequence\"} # feature -> processor\n", + " output_schema = {\"label\": \"binary\"} # label -> processor\n", + "\n", + " def __call__(self, patient) -> list[dict]: # one patient -> samples\n", + " ...\n", + "```\n", + "\n", + "The **schemas** name a *processor* for each field. Processors turn raw\n", + "values (lists of codes, numbers, text) into tensors, learning what they\n", + "need (such as a code vocabulary) from the data. Think of them as the\n", + "tokenizer of a language model, applied to each field." + ] + }, + { + "cell_type": "markdown", + "id": "cell-04", + "metadata": {}, + "source": [ + "## Loading the data\n", + "\n", + "Same dataset as Tutorial 01, without the notes table." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-05", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.datasets import MIMIC3Dataset\n", + "\n", + "dataset = MIMIC3Dataset(\n", + " root=\"https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III\",\n", + " tables=[\"diagnoses_icd\", \"procedures_icd\", \"prescriptions\"],\n", + " cache_dir=\"pyhealth_cache\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-06", + "metadata": {}, + "source": [ + "## Processors, one at a time\n", + "\n", + "Before writing a task, let's see what the common processors do. We\n", + "build a toy dataset of three hand-written samples with\n", + "`create_sample_dataset`, which applies the processors exactly as\n", + "`set_task` does, then print what came out." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-07", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.datasets import create_sample_dataset\n", + "\n", + "toy = [\n", + " {\"patient_id\": \"a\", \"codes\": [\"I10\", \"E11\"], \"visits\": [[\"I10\"], [\"E11\", \"N18\"]],\n", + " \"flags\": [\"female\", \"smoker\"], \"labs\": [1.2, 140.0], \"label\": 1},\n", + " {\"patient_id\": \"b\", \"codes\": [\"J45\"], \"visits\": [[\"J45\"]],\n", + " \"flags\": [\"male\"], \"labs\": [0.9, 138.0], \"label\": 0},\n", + " {\"patient_id\": \"c\", \"codes\": [\"I10\", \"J45\", \"N18\"], \"visits\": [[\"I10\", \"J45\"], [\"N18\"], [\"I10\"]],\n", + " \"flags\": [\"female\"], \"labs\": [1.5, 135.0], \"label\": 1},\n", + "]\n", + "toy_ds = create_sample_dataset(\n", + " samples=toy,\n", + " input_schema={\"codes\": \"sequence\", \"visits\": \"nested_sequence\",\n", + " \"flags\": \"multi_hot\", \"labs\": \"tensor\"},\n", + " output_schema={\"label\": \"binary\"},\n", + ")\n", + "for key, value in toy_ds[2].items():\n", + " print(f\"{key:8s} {value}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-08", + "metadata": {}, + "source": [ + "Reading the output for patient `c`:\n", + "\n", + "| Field | Processor | Raw value | Becomes |\n", + "|---|---|---|---|\n", + "| `codes` | `sequence` | a list of codes | one integer id per code (ids 0 and 1 are reserved for padding and unknown codes) |\n", + "| `visits` | `nested_sequence` | a list of visits, each a list of codes | a 2-D tensor, one row per visit, padded to the longest visit |\n", + "| `flags` | `multi_hot` | a set of categories | a 0/1 vector over all categories seen |\n", + "| `labs` | `tensor` | numbers | a float tensor, unchanged |\n", + "| `label` | `binary` | 0/1 | a float tensor of shape (1,) |\n", + "\n", + "The vocabularies the processors learned are kept with the dataset, so\n", + "you can always map ids back to codes:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-09", + "metadata": {}, + "outputs": [], + "source": [ + "codes_vocab = toy_ds.input_processors[\"codes\"].code_vocab\n", + "print(\"codes vocabulary:\", codes_vocab)\n", + "print(\"flags categories:\", toy_ds.input_processors[\"flags\"].label_vocab)\n", + "\n", + "id_to_code = {i: c for c, i in codes_vocab.items()}\n", + "print(\"patient c's codes:\", [id_to_code[int(i)] for i in toy_ds[2][\"codes\"]])" + ] + }, + { + "cell_type": "markdown", + "id": "cell-10", + "metadata": {}, + "source": [ + "Other processors you will meet:\n", + "\n", + "| Processor | Use it for |\n", + "|---|---|\n", + "| `multiclass`, `multilabel`, `regression` | labels with several classes, several labels at once, or a number |\n", + "| `timeseries` | irregular measurements resampled to a fixed grid |\n", + "| `text` | free text, for language models |\n", + "| `nested_multihot` | per-visit code sets as 0/1 rows (compact for large vocabularies) |\n", + "| `image`, `audio`, `signal` | file paths to images, audio, physiological signals |\n", + "| `raw` | values a model reads as Python objects (kept unchanged) |\n", + "\n", + "The full list is in the\n", + "[processors docs](https://pyhealth.readthedocs.io/en/latest/api/processors.html).\n", + "\n", + "## Using a built-in task\n", + "\n", + "PyHealth ships tasks for common questions on each dataset: mortality,\n", + "readmission, length of stay, drug recommendation, medical coding and\n", + "more. Built-in tasks can take options; readmission, for example, lets\n", + "you choose the time window and whether to drop admissions of minors.\n", + "\n", + "Options matter: by default this task skips patients under 18. The\n", + "synthetic data shifts every birth date to just before the first\n", + "admission, so every patient looks like a newborn and the default would\n", + "return no samples at all. On real MIMIC-III keep the default; here we\n", + "turn it off. When a task returns nothing, check its options and the\n", + "values it reads before suspecting the data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-11", + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import timedelta\n", + "from pyhealth.tasks import ReadmissionPredictionMIMIC3\n", + "\n", + "readmission = ReadmissionPredictionMIMIC3(window=timedelta(days=30), exclude_minors=False)\n", + "print(readmission.input_schema, \"->\", readmission.output_schema)\n", + "\n", + "readmit_samples = dataset.set_task(readmission)\n", + "positives = sum(int(readmit_samples[i][\"readmission\"]) for i in range(len(readmit_samples)))\n", + "print(f\"{len(readmit_samples)} samples, {positives} readmitted within 30 days\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-12", + "metadata": {}, + "source": [ + "## Writing your own task\n", + "\n", + "Suppose we want to predict a **long hospital stay** (more than 7 days)\n", + "at admission time, from:\n", + "\n", + "- the diagnosis codes of the current admission (`sequence`),\n", + "- the diagnosis codes of every admission so far, visit by visit\n", + " (`nested_sequence`), so a model can see the history,\n", + "- the admission type and the patient's sex (`multi_hot`),\n", + "- the number of earlier admissions (`tensor`).\n", + "\n", + "The `__call__` method is plain Python over the patient's events, using\n", + "the same `get_events` calls as Tutorial 01.\n", + "\n", + "`pre_filter` is optional. It runs once on the whole event table before\n", + "any patient is processed, so it is a cheap place to drop patients the\n", + "task can never use. Here we keep patients with at least one diagnosis." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-13", + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "\n", + "import polars as pl\n", + "from pyhealth.tasks import BaseTask\n", + "\n", + "\n", + "class LongStayMIMIC3(BaseTask):\n", + " \"\"\"Predicts whether an admission lasts more than 7 days.\"\"\"\n", + "\n", + " task_name = \"LongStayMIMIC3\"\n", + " input_schema = {\n", + " \"conditions\": \"sequence\",\n", + " \"history\": \"nested_sequence\",\n", + " \"demographics\": \"multi_hot\",\n", + " \"prior_admissions\": \"tensor\",\n", + " }\n", + " output_schema = {\"long_stay\": \"binary\"}\n", + "\n", + " def pre_filter(self, df: pl.LazyFrame) -> pl.LazyFrame:\n", + " with_diagnoses = (\n", + " df.filter(pl.col(\"event_type\") == \"diagnoses_icd\").select(\"patient_id\").unique()\n", + " )\n", + " return df.join(with_diagnoses, on=\"patient_id\", how=\"semi\")\n", + "\n", + " def __call__(self, patient):\n", + " sex = patient.get_events(event_type=\"patients\")[0].gender\n", + " history = []\n", + " samples = []\n", + " for n, adm in enumerate(patient.get_events(event_type=\"admissions\")):\n", + " codes = [e.icd9_code for e in patient.get_events(\n", + " event_type=\"diagnoses_icd\", filters=[(\"hadm_id\", \"==\", adm.hadm_id)])]\n", + " if not codes or adm.dischtime is None:\n", + " continue\n", + " history.append(codes)\n", + " discharge = datetime.strptime(adm.dischtime, \"%Y-%m-%d %H:%M:%S\")\n", + " stay_days = (discharge - adm.timestamp).total_seconds() / 86400\n", + " samples.append({\n", + " \"patient_id\": patient.patient_id,\n", + " \"visit_id\": adm.hadm_id,\n", + " \"conditions\": codes,\n", + " \"history\": list(history),\n", + " \"demographics\": [f\"sex={sex}\", f\"type={adm.admission_type}\"],\n", + " \"prior_admissions\": [float(n)],\n", + " \"long_stay\": int(stay_days > 7),\n", + " })\n", + " return samples" + ] + }, + { + "cell_type": "markdown", + "id": "cell-14", + "metadata": {}, + "source": [ + "Two habits worth keeping in your own tasks:\n", + "\n", + "- **Only use information available at prediction time.** The inputs\n", + " above come from the current and earlier admissions; the label comes\n", + " from the discharge time, which is in the future when the prediction\n", + " is made. A feature such as \"number of drugs given during the stay\"\n", + " would leak the answer.\n", + "- **Test on one patient first.** A task is just a function, so call it\n", + " directly before running it on everyone:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-15", + "metadata": {}, + "outputs": [], + "source": [ + "task = LongStayMIMIC3()\n", + "for pid in dataset.unique_patient_ids[:200]:\n", + " out = task(dataset.get_patient(pid))\n", + " if len(out) > 1:\n", + " for s in out:\n", + " print({k: s[k] for k in (\"visit_id\", \"conditions\", \"demographics\", \"prior_admissions\", \"long_stay\")})\n", + " break" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-16", + "metadata": {}, + "outputs": [], + "source": [ + "long_stay = dataset.set_task(task)\n", + "print(f\"{len(long_stay)} samples\")\n", + "sample = long_stay[0]\n", + "for key in task.input_schema | task.output_schema:\n", + " print(f\"{key:17s} {tuple(sample[key].shape)} {sample[key]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-17", + "metadata": {}, + "source": [ + "## The task works with any model\n", + "\n", + "Because the schemas describe the inputs, models need no extra\n", + "configuration. Here `MultimodalRNN` routes the sequence fields through\n", + "recurrent layers and the multi-hot and numeric fields through linear\n", + "layers. (Tutorial 03 covers training properly; this is a smoke test.)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-18", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.datasets import get_dataloader, split_by_patient\n", + "from pyhealth.models import MultimodalRNN\n", + "from pyhealth.trainer import Trainer\n", + "\n", + "train_ds, val_ds, test_ds = split_by_patient(long_stay, [0.7, 0.1, 0.2], seed=0)\n", + "trainer = Trainer(model=MultimodalRNN(dataset=long_stay), metrics=[\"roc_auc\", \"pr_auc\"])\n", + "trainer.train(\n", + " train_dataloader=get_dataloader(train_ds, batch_size=64, shuffle=True),\n", + " val_dataloader=get_dataloader(val_ds, batch_size=64),\n", + " epochs=3,\n", + " monitor=\"roc_auc\",\n", + ")\n", + "print(trainer.evaluate(get_dataloader(test_ds, batch_size=64)))" + ] + }, + { + "cell_type": "markdown", + "id": "cell-19", + "metadata": {}, + "source": [ + "## Tasks beyond patient timelines\n", + "\n", + "A \"patient\" is just whatever a dataset groups events by. For image or\n", + "document collections, each row can be its own patient, and the task\n", + "returns one sample per row. PyHealth's chest X-ray task, for example,\n", + "is only a few lines:\n", + "\n", + "```python\n", + "class COVID19CXRClassification(BaseTask):\n", + " task_name = \"COVID19CXRClassification\"\n", + " input_schema = {\"image\": \"image\"} # a file path -> image tensor\n", + " output_schema = {\"disease\": \"multiclass\"}\n", + "\n", + " def __call__(self, patient):\n", + " event = patient.get_events(event_type=\"covid19_cxr\")[0]\n", + " return [{\"image\": event.path, \"disease\": event.label}]\n", + "```\n", + "\n", + "Tutorial 07 uses the same pattern for clinical text.\n", + "\n", + "## Summary\n", + "\n", + "- A task maps one patient to a list of sample dictionaries.\n", + "- `input_schema` / `output_schema` choose a processor per field; the\n", + " processors learn vocabularies and produce tensors.\n", + "- `pre_filter` drops unusable patients early; call the task on one\n", + " patient to debug it.\n", + "- Keep features to what is known at prediction time." + ] + }, + { + "cell_type": "markdown", + "id": "cell-20", + "metadata": {}, + "source": [ + "## Where to go next\n", + "\n", + "**Next:** Tutorial 03, *Training and evaluating models properly*.\n", + "\n", + "The whole series:\n", + "\n", + "00. Quickstart: your first clinical prediction model\n", + "01. Datasets: patients, events and your own data\n", + "02. Tasks and processors: from patients to model-ready samples\n", + "03. Training and evaluating models properly\n", + "04. Choosing a model: from logistic regression to XGBoost\n", + "05. Interpreting predictions\n", + "06. Medical codes: lookups, mappings and groupers\n", + "07. Clinical text: classification and medical coding\n", + "08. Contributing a dataset or task to PyHealth\n", + "\n", + "Questions or problems? Open an issue at https://github.com/sunlabuiuc/PyHealth/issues. If PyHealth is useful to you, a star on GitHub helps the project." + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/tutorials/series/03_training_and_evaluation.ipynb b/examples/tutorials/series/03_training_and_evaluation.ipynb new file mode 100644 index 000000000..64c7f753e --- /dev/null +++ b/examples/tutorials/series/03_training_and_evaluation.ipynb @@ -0,0 +1,413 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-00", + "metadata": {}, + "source": [ + "# PyHealth Tutorial 03: Training and evaluating models properly\n", + "\n", + "**What you will learn**\n", + "- How to split so that nothing about the test patients leaks into training\n", + "- How to control training: epochs, early stopping, learning rate, checkpoints\n", + "- How to handle a rare outcome with a positive-class weight, and what it costs\n", + "- How to report discrimination and calibration with patient-level confidence intervals\n", + "\n", + "**Time:** about 25 minutes on a free Colab CPU runtime. \n", + "**Before this:** Tutorials 00 and 02.\n", + "\n", + "All data used here is synthetic or public, so every cell runs without credentials." + ] + }, + { + "cell_type": "markdown", + "id": "cell-01", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub. Once 2.1 is on PyPI this becomes `pip install \"pyhealth>=2.1\"`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-02", + "metadata": { + "tags": [ + "install" + ] + }, + "outputs": [], + "source": [ + "%pip install -q \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"" + ] + }, + { + "cell_type": "markdown", + "id": "cell-03", + "metadata": {}, + "source": [ + "Getting a number out of `trainer.evaluate` is easy. Getting a number\n", + "you can trust, and report, takes a few more steps. This tutorial\n", + "walks through them on the in-hospital mortality task from Tutorial 00,\n", + "a rare outcome (about 3% of samples) where the details matter most." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-04", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.datasets import MIMIC3Dataset\n", + "from pyhealth.tasks import MortalityPredictionMIMIC3\n", + "\n", + "dataset = MIMIC3Dataset(\n", + " root=\"https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III\",\n", + " tables=[\"diagnoses_icd\", \"procedures_icd\", \"prescriptions\"],\n", + " cache_dir=\"pyhealth_cache\",\n", + ")\n", + "task = MortalityPredictionMIMIC3()" + ] + }, + { + "cell_type": "markdown", + "id": "cell-05", + "metadata": {}, + "source": [ + "## 1. Split by patient, and fit preprocessing on training patients only\n", + "\n", + "Tutorial 00 already split **by patient**, so no patient appears in two\n", + "parts. There is a second, quieter leak: the processors. When you call\n", + "`set_task(task)` and split afterwards, the code vocabularies are built\n", + "from *all* samples, including the test patients. The model then has an\n", + "embedding slot for a code that only test patients have, which a real\n", + "deployment would never have.\n", + "\n", + "Passing `split=PatientSplit(...)` to `set_task` splits first and fits\n", + "every processor on the training patients only. It returns one dataset\n", + "per part, training part first. Codes that appear only in validation or\n", + "test map to ``, exactly as unseen codes would in practice." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-06", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.datasets import PatientSplit\n", + "\n", + "train_ds, val_ds, test_ds = dataset.set_task(\n", + " task, split=PatientSplit(ratios=(0.7, 0.1, 0.2), seed=42)\n", + ")\n", + "print(f\"train {len(train_ds)} val {len(val_ds)} test {len(test_ds)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-07", + "metadata": {}, + "source": [ + "Compare with fitting on everything first. The vocabulary fitted on all\n", + "samples is larger: the extra entries are codes seen only in the\n", + "validation and test patients." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-08", + "metadata": {}, + "outputs": [], + "source": [ + "all_samples = dataset.set_task(task)\n", + "\n", + "for field in [\"conditions\", \"procedures\", \"drugs\"]:\n", + " v_all = len(all_samples.input_processors[field].code_vocab)\n", + " v_train = len(train_ds.input_processors[field].code_vocab)\n", + " print(f\"{field:11s} vocabulary fitted on all samples {v_all:5d}, on training patients {v_train:5d}\")\n", + "\n", + "unk = train_ds.input_processors[\"conditions\"].code_vocab[\"\"]\n", + "test_codes = [int(c) for i in range(len(test_ds)) for c in test_ds[i][\"conditions\"]]\n", + "print(f\"{sum(c == unk for c in test_codes) / len(test_codes):.1%} of test diagnosis codes are unseen in training\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-09", + "metadata": {}, + "source": [ + "For processors that learn statistics, such as normalisation, the same\n", + "rule applies: statistics from test patients must not shape the\n", + "training inputs. With `split=` you get that for free.\n", + "\n", + "## 2. Training with more control\n", + "\n", + "`Trainer.train` has a few options worth knowing:\n", + "\n", + "| Option | What it does |\n", + "|---|---|\n", + "| `epochs` | Maximum passes over the training data |\n", + "| `monitor`, `monitor_criterion` | Validation metric that picks the best epoch (`\"max\"` or `\"min\"`) |\n", + "| `patience` | Stop early after this many epochs without improvement |\n", + "| `optimizer_params` | e.g. `{\"lr\": 1e-3}` |\n", + "| `weight_decay`, `max_grad_norm` | Regularisation and gradient clipping |\n", + "\n", + "The `Trainer` itself takes `output_path` and `exp_name`; the best and\n", + "last checkpoints are saved there, and the best one is reloaded at the\n", + "end of training." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-10", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.datasets import get_dataloader\n", + "from pyhealth.models import RNN\n", + "from pyhealth.trainer import Trainer\n", + "\n", + "train_loader = get_dataloader(train_ds, batch_size=64, shuffle=True)\n", + "val_loader = get_dataloader(val_ds, batch_size=256)\n", + "test_loader = get_dataloader(test_ds, batch_size=256)\n", + "\n", + "METRICS = [\"pr_auc\", \"roc_auc\"]\n", + "\n", + "model = RNN(dataset=train_ds)\n", + "trainer = Trainer(model=model, metrics=METRICS, output_path=\"runs\", exp_name=\"rnn_plain\")\n", + "trainer.train(\n", + " train_dataloader=train_loader,\n", + " val_dataloader=val_loader,\n", + " epochs=20,\n", + " monitor=\"pr_auc\",\n", + " patience=5,\n", + " optimizer_params={\"lr\": 1e-3},\n", + " weight_decay=1e-4,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-11", + "metadata": {}, + "source": [ + "## 3. Rare outcomes: weighting the positive class\n", + "\n", + "With 3% positives, the loss is dominated by the negatives and a model\n", + "can do well by predicting low risk for everyone. A common remedy is to\n", + "weight each positive example more. `set_pos_weight(\"balanced\",\n", + "train_ds)` sets the weight to negatives / positives in the **training**\n", + "data (never the full dataset, so test labels do not set it)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-12", + "metadata": {}, + "outputs": [], + "source": [ + "weighted = RNN(dataset=train_ds)\n", + "weighted.set_pos_weight(\"balanced\", train_ds)\n", + "print(f\"positive weight: {float(weighted.pos_weight[0]):.1f}\")\n", + "\n", + "trainer_w = Trainer(model=weighted, metrics=METRICS, output_path=\"runs\", exp_name=\"rnn_weighted\")\n", + "trainer_w.train(\n", + " train_dataloader=train_loader,\n", + " val_dataloader=val_loader,\n", + " epochs=20,\n", + " monitor=\"pr_auc\",\n", + " patience=5,\n", + " optimizer_params={\"lr\": 1e-3},\n", + " weight_decay=1e-4,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-13", + "metadata": {}, + "source": [ + "## 4. Discrimination *and* calibration\n", + "\n", + "Ranking patients well (discrimination: ROC-AUC, PR-AUC) is half of\n", + "the story. If the predicted risks will be read as probabilities (\"this\n", + "patient has a 20% risk\"), they also need to be **calibrated**:\n", + "\n", + "| Metric | Ideal | Reading it |\n", + "|---|---|---|\n", + "| `brier` | 0 | mean squared error of the probabilities |\n", + "| `oe_ratio` | 1 | observed events / expected events; below 1 means risks are overestimated |\n", + "| `calibration_slope` | 1 | below 1 means predictions are too extreme |\n", + "| `calibration_intercept` | 0 | above 0 means risks are underestimated overall |\n", + "\n", + "`Trainer.inference` returns the labels and predicted probabilities;\n", + "`binary_metrics_fn` scores them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-14", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from pyhealth.metrics import binary_metrics_fn\n", + "\n", + "REPORT = [\"pr_auc\", \"roc_auc\", \"brier\", \"oe_ratio\", \"calibration_slope\", \"calibration_intercept\"]\n", + "\n", + "y_true, prob_plain, _, pids = trainer.inference(test_loader, return_patient_ids=True)\n", + "_, prob_weighted, _ = trainer_w.inference(test_loader)\n", + "y_true, prob_plain, prob_weighted = y_true.ravel(), prob_plain.ravel(), prob_weighted.ravel()\n", + "\n", + "print(f\"test prevalence {y_true.mean():.3f}\\n\")\n", + "print(f\"{'metric':22s} {'plain':>8s} {'weighted':>9s}\")\n", + "plain = binary_metrics_fn(y_true, prob_plain, metrics=REPORT)\n", + "weighted_scores = binary_metrics_fn(y_true, prob_weighted, metrics=REPORT)\n", + "for m in REPORT:\n", + " print(f\"{m:22s} {plain[m]:8.3f} {weighted_scores[m]:9.3f}\")\n", + "print(f\"\\nmean predicted risk: plain {prob_plain.mean():.3f}, weighted {prob_weighted.mean():.3f}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-15", + "metadata": {}, + "source": [ + "Look at the mean predicted risk and the observed / expected ratio:\n", + "weighting pushes every probability up, so the weighted model's risks\n", + "are inflated even when its ranking is similar. That is the price of\n", + "weighting. If you need calibrated risks, either train without the\n", + "weight, or recalibrate on a separate calibration set of patients.\n", + "\n", + "## 5. Confidence intervals that respect patients\n", + "\n", + "A test set of a few hundred samples gives noisy metrics, so report an\n", + "interval. Samples from the same patient are correlated, so the\n", + "bootstrap should resample **whole patients** (`groups=patient_ids`),\n", + "not individual samples. `paired_bootstrap_diff` compares two models on\n", + "the same resampled patients, which is the right way to ask \"is model A\n", + "better than model B?\"." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-16", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.metrics import bootstrap_ci, paired_bootstrap_diff\n", + "\n", + "groups = np.asarray(pids)\n", + "for name, prob in [(\"plain\", prob_plain), (\"weighted\", prob_weighted)]:\n", + " for metric in [\"roc_auc\", \"brier\"]:\n", + " ci = bootstrap_ci(y_true, prob, metric, groups=groups, n_boot=500, seed=0)\n", + " print(f\"{name:9s} {metric:8s} {ci['estimate']:.3f} 95% CI [{ci['lower']:.3f}, {ci['upper']:.3f}]\")\n", + "\n", + "diff = paired_bootstrap_diff(y_true, prob_weighted, prob_plain, \"roc_auc\", groups=groups, n_boot=500, seed=0)\n", + "print(f\"\\nweighted - plain ROC-AUC {diff['estimate']:+.3f} 95% CI [{diff['lower']:+.3f}, {diff['upper']:+.3f}]\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-17", + "metadata": {}, + "source": [ + "If the interval of the difference includes 0, the data cannot tell the\n", + "two models apart. With a few hundred test samples and a 3% outcome,\n", + "that is the usual result, and worth knowing before claiming a\n", + "winner.\n", + "\n", + "## 6. Saving and reloading a model\n", + "\n", + "The trainer already saved the best checkpoint under\n", + "`runs//best.ckpt`. To reuse a model later, build the same\n", + "architecture on a dataset with the same processors, and load it:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-18", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "ckpt = os.path.join(trainer.exp_path, \"best.ckpt\")\n", + "restored = Trainer(model=RNN(dataset=train_ds), metrics=METRICS, checkpoint_path=ckpt)\n", + "print(\"restored:\", restored.evaluate(test_loader))\n", + "print(\"original:\", trainer.evaluate(test_loader))" + ] + }, + { + "cell_type": "markdown", + "id": "cell-19", + "metadata": {}, + "source": [ + "To use the model on new data later, also keep the fitted processors\n", + "(`pyhealth.datasets.save_processors` / `load_processors`), so new\n", + "samples get the same code ids.\n", + "\n", + "## Summary\n", + "\n", + "- Use `set_task(task, split=PatientSplit(...))`: patients stay in one\n", + " part, and preprocessing learns from training patients only.\n", + "- Pick the best epoch on validation with `monitor` and `patience`;\n", + " touch the test set once.\n", + "- A positive-class weight can help rare outcomes but inflates\n", + " probabilities; check calibration.\n", + "- Report discrimination and calibration with patient-level bootstrap\n", + " intervals, and compare models with a paired bootstrap." + ] + }, + { + "cell_type": "markdown", + "id": "cell-20", + "metadata": {}, + "source": [ + "## Where to go next\n", + "\n", + "**Next:** Tutorial 04, *Choosing a model: from logistic regression to XGBoost*.\n", + "\n", + "The whole series:\n", + "\n", + "00. Quickstart: your first clinical prediction model\n", + "01. Datasets: patients, events and your own data\n", + "02. Tasks and processors: from patients to model-ready samples\n", + "03. Training and evaluating models properly\n", + "04. Choosing a model: from logistic regression to XGBoost\n", + "05. Interpreting predictions\n", + "06. Medical codes: lookups, mappings and groupers\n", + "07. Clinical text: classification and medical coding\n", + "08. Contributing a dataset or task to PyHealth\n", + "\n", + "Questions or problems? Open an issue at https://github.com/sunlabuiuc/PyHealth/issues. If PyHealth is useful to you, a star on GitHub helps the project." + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/tutorials/series/04_choosing_a_model.ipynb b/examples/tutorials/series/04_choosing_a_model.ipynb new file mode 100644 index 000000000..265ca85fc --- /dev/null +++ b/examples/tutorials/series/04_choosing_a_model.ipynb @@ -0,0 +1,375 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-00", + "metadata": {}, + "source": [ + "# PyHealth Tutorial 04: Choosing a model: from logistic regression to XGBoost\n", + "\n", + "**What you will learn**\n", + "- What the main PyHealth models assume about the data, and when to pick each\n", + "- How to compare several models fairly on the same patients\n", + "- How to add a gradient-boosted tree baseline (XGBoost) to a PyHealth benchmark\n", + "\n", + "**Time:** about 20 minutes on a free Colab CPU runtime. \n", + "**Before this:** Tutorial 03.\n", + "\n", + "All data used here is synthetic or public, so every cell runs without credentials." + ] + }, + { + "cell_type": "markdown", + "id": "cell-01", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub. Once 2.1 is on PyPI this becomes `pip install \"pyhealth>=2.1\"`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-02", + "metadata": { + "tags": [ + "install" + ] + }, + "outputs": [], + "source": [ + "%pip install -q \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"" + ] + }, + { + "cell_type": "markdown", + "id": "cell-03", + "metadata": {}, + "source": [ + "## The models at a glance\n", + "\n", + "All PyHealth models take the dataset in their constructor and read the\n", + "input and output schemas from it, so swapping models is a one-line\n", + "change. The main ones for coded EHR data:\n", + "\n", + "| Model | Idea | Good first choice when |\n", + "|---|---|---|\n", + "| `LogisticRegression` | Embeds each code, sums them, one linear layer | You want a transparent, hard-to-beat baseline |\n", + "| `MLP` | Same pooled embeddings, then hidden layers | Features interact, but order does not matter |\n", + "| `RNN` | Reads each field as a sequence (GRU or LSTM) | Order within a field carries signal |\n", + "| `Transformer` | Self-attention over each field | Longer histories, long-range interactions |\n", + "| `RETAIN` | Two attention layers over visits | You need visit- and code-level attention weights |\n", + "| `XGBoostModel` | Gradient-boosted trees on code counts or tabular features | Tabular or bag-of-codes data; the standard strong baseline |\n", + "\n", + "On tabular and bag-of-codes data, gradient-boosted trees are often as\n", + "good as neural models and much faster to train, so a benchmark without\n", + "them can overstate what a neural model adds.\n", + "\n", + "## The task: spotting heart failure from the medication list\n", + "\n", + "We need a question the synthetic data can answer, so we use a\n", + "*phenotyping* task: does this admission include a heart-failure\n", + "diagnosis (ICD-9 428.x), judged from the drugs and procedures recorded\n", + "in it? Phenotyping models like this help find patients with a\n", + "condition when diagnosis codes are missing or unreliable, for example\n", + "when building a study cohort. About 1 in 5 admissions qualifies.\n", + "\n", + "The task is a few lines (Tutorial 02 explains each part). Drugs are\n", + "kept by name so the results stay readable." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-04", + "metadata": {}, + "outputs": [], + "source": [ + "from collections import defaultdict\n", + "\n", + "from pyhealth.tasks import BaseTask\n", + "\n", + "\n", + "class HeartFailureFromMedications(BaseTask):\n", + " \"\"\"Does an admission include a heart-failure diagnosis (ICD-9 428.x)?\"\"\"\n", + "\n", + " task_name = \"HeartFailureFromMedications\"\n", + " input_schema = {\"drugs\": \"sequence\", \"procedures\": \"sequence\"}\n", + " output_schema = {\"heart_failure\": \"binary\"}\n", + "\n", + " def __call__(self, patient):\n", + " # Fetch each event type once, then group by admission: much\n", + " # faster than one filtered query per admission.\n", + " by_admission = defaultdict(lambda: {\"drugs\": [], \"diagnoses\": [], \"procedures\": []})\n", + " for e in patient.get_events(event_type=\"prescriptions\"):\n", + " if e.drug:\n", + " by_admission[e.hadm_id][\"drugs\"].append(e.drug)\n", + " for e in patient.get_events(event_type=\"diagnoses_icd\"):\n", + " by_admission[e.hadm_id][\"diagnoses\"].append(e.icd9_code)\n", + " for e in patient.get_events(event_type=\"procedures_icd\"):\n", + " by_admission[e.hadm_id][\"procedures\"].append(e.icd9_code)\n", + "\n", + " samples = []\n", + " for hadm_id, codes in by_admission.items():\n", + " if not codes[\"drugs\"] or not codes[\"diagnoses\"]:\n", + " continue\n", + " samples.append({\n", + " \"patient_id\": patient.patient_id,\n", + " \"visit_id\": hadm_id,\n", + " \"drugs\": codes[\"drugs\"],\n", + " \"procedures\": codes[\"procedures\"] or [\"none\"],\n", + " \"heart_failure\": int(any(c.startswith(\"428\") for c in codes[\"diagnoses\"])),\n", + " })\n", + " return samples" + ] + }, + { + "cell_type": "markdown", + "id": "cell-05", + "metadata": {}, + "source": [ + "## One split for every model\n", + "\n", + "A fair comparison uses the same patients for every model, so we build\n", + "the samples once, split by patient (Tutorial 03), and reuse the three\n", + "parts." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-06", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.datasets import MIMIC3Dataset, PatientSplit, get_dataloader\n", + "\n", + "dataset = MIMIC3Dataset(\n", + " root=\"https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III\",\n", + " tables=[\"diagnoses_icd\", \"procedures_icd\", \"prescriptions\"],\n", + " cache_dir=\"pyhealth_cache\",\n", + ")\n", + "train_ds, val_ds, test_ds = dataset.set_task(\n", + " HeartFailureFromMedications(), split=PatientSplit((0.7, 0.1, 0.2), seed=7)\n", + ")\n", + "print(f\"train {len(train_ds)} val {len(val_ds)} test {len(test_ds)}\")\n", + "\n", + "val_loader = get_dataloader(val_ds, batch_size=512)\n", + "test_loader = get_dataloader(test_ds, batch_size=512)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-07", + "metadata": {}, + "source": [ + "## Neural models\n", + "\n", + "One helper trains any model the same way: same optimizer, same\n", + "maximum epochs, the best epoch picked on validation PR-AUC, and the\n", + "same test set. Only the model changes.\n", + "\n", + "Sequence models are slow on a CPU with 35,000 training admissions, so\n", + "on a CPU runtime we train the three fastest; on a GPU runtime\n", + "(Runtime > Change runtime type) all five run in a few minutes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-08", + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "\n", + "import numpy as np\n", + "import torch\n", + "from pyhealth.models import MLP, RETAIN, RNN, LogisticRegression, Transformer\n", + "from pyhealth.trainer import Trainer\n", + "\n", + "METRICS = [\"pr_auc\", \"roc_auc\"]\n", + "results = {}\n", + "\n", + "\n", + "def fit_and_test(name, model_class, **kwargs):\n", + " torch.manual_seed(0)\n", + " model = model_class(dataset=train_ds, **kwargs)\n", + " trainer = Trainer(model=model, metrics=METRICS, output_path=\"runs\", exp_name=name)\n", + " start = time.time()\n", + " trainer.train(\n", + " train_dataloader=get_dataloader(train_ds, batch_size=256, shuffle=True),\n", + " val_dataloader=val_loader,\n", + " epochs=6,\n", + " monitor=\"pr_auc\",\n", + " patience=2,\n", + " )\n", + " scores = trainer.evaluate(test_loader)\n", + " n_params = sum(p.numel() for p in model.parameters())\n", + " results[name] = {**scores, \"seconds\": time.time() - start, \"parameters\": n_params}\n", + " return trainer\n", + "\n", + "\n", + "models = [(\"LogisticRegression\", LogisticRegression), (\"MLP\", MLP), (\"RNN\", RNN)]\n", + "if torch.cuda.is_available():\n", + " models += [(\"Transformer\", Transformer), (\"RETAIN\", RETAIN)]\n", + "else:\n", + " print(\"CPU runtime: skipping Transformer and RETAIN (about 10 minutes each on CPU). \"\n", + " \"Switch to a GPU runtime to include them.\")\n", + "for name, cls in models:\n", + " fit_and_test(name, cls)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-09", + "metadata": {}, + "source": [ + "## Adding XGBoost\n", + "\n", + "Gradient-boosted trees do not train with `Trainer.train`: they are fit\n", + "once on the full training set with `model.fit(...)`. After that they\n", + "work with the same `Trainer.evaluate`, metrics and checkpoints as\n", + "every other model.\n", + "\n", + "Trees need fixed-width numeric inputs. Code lists vary in length, so\n", + "`bag_of_codes=True` turns each field into per-sample code counts over\n", + "the vocabulary (one column per code). `early_stopping_rounds` uses the\n", + "validation set to choose the number of trees.\n", + "\n", + "Use unshuffled loaders for `fit`: the order of rows affects row\n", + "subsampling, so a fixed order makes the fit reproducible." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-10", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.models import XGBoostModel\n", + "\n", + "start = time.time()\n", + "xgb = XGBoostModel(\n", + " train_ds,\n", + " bag_of_codes=True,\n", + " n_estimators=500,\n", + " max_depth=4,\n", + " learning_rate=0.05,\n", + " subsample=0.8,\n", + " colsample_bytree=0.8,\n", + " random_state=0,\n", + " early_stopping_rounds=50,\n", + " eval_metric=\"aucpr\",\n", + ")\n", + "xgb.fit(get_dataloader(train_ds, batch_size=1024), val_loader)\n", + "scores = Trainer(model=xgb, metrics=METRICS, enable_logging=False).evaluate(test_loader)\n", + "results[\"XGBoost\"] = {**scores, \"seconds\": time.time() - start,\n", + " \"parameters\": xgb.estimators_[0].best_iteration + 1}\n", + "print(f\"{len(xgb.feature_names)} columns, e.g. {xgb.feature_names[:3]}\")\n", + "print(f\"best number of trees: {xgb.estimators_[0].best_iteration + 1}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-11", + "metadata": {}, + "source": [ + "## Results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-12", + "metadata": {}, + "outputs": [], + "source": [ + "prevalence = float(np.mean([int(test_ds[i][\"heart_failure\"]) for i in range(len(test_ds))]))\n", + "print(f\"test prevalence (PR-AUC of a random guess): {prevalence:.3f}\\n\")\n", + "print(f\"{'model':20s} {'PR-AUC':>7s} {'ROC-AUC':>8s} {'seconds':>8s} {'size':>10s}\")\n", + "for name, r in sorted(results.items(), key=lambda kv: -kv[1][\"pr_auc\"]):\n", + " size = f\"{r['parameters']} trees\" if name == \"XGBoost\" else f\"{r['parameters']:,}\"\n", + " print(f\"{name:20s} {r['pr_auc']:7.3f} {r['roc_auc']:8.3f} {r['seconds']:8.1f} {size:>10s}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-13", + "metadata": {}, + "source": [ + "### Reading the table\n", + "\n", + "- Compare every model with the random-guess line first: PR-AUC is only\n", + " meaningful relative to the prevalence.\n", + "- Look at cost as well as score: training time and model size differ\n", + " by orders of magnitude for similar accuracy.\n", + "- Small gaps can be noise. Before\n", + " declaring a winner, compute bootstrap intervals of the paired\n", + " difference (Tutorial 03, `paired_bootstrap_diff`).\n", + "- Neural model scores also move with the random seed. Report the mean\n", + " and spread over several seeds rather than one run.\n", + "\n", + "## Rules of thumb\n", + "\n", + "1. Start with `LogisticRegression` and `XGBoostModel`. They are fast\n", + " and often strong, and every other model has to beat them.\n", + "2. Move to `RNN`, `Transformer` or `RETAIN` when order or long\n", + " histories plausibly matter, and check the gain with intervals.\n", + "3. Keep the split, the metric used to pick the epoch, and the test set\n", + " identical across models.\n", + "\n", + "## Summary\n", + "\n", + "- Every model reads the schemas, so swapping models is one line.\n", + "- `XGBoostModel` fits with `model.fit(train, val)` and then works with\n", + " the standard `Trainer` evaluation; `bag_of_codes=True` handles code\n", + " lists.\n", + "- Compare on the same patients, report intervals, and include strong\n", + " simple baselines." + ] + }, + { + "cell_type": "markdown", + "id": "cell-14", + "metadata": {}, + "source": [ + "## Where to go next\n", + "\n", + "**Next:** Tutorial 05, *Interpreting predictions*.\n", + "\n", + "The whole series:\n", + "\n", + "00. Quickstart: your first clinical prediction model\n", + "01. Datasets: patients, events and your own data\n", + "02. Tasks and processors: from patients to model-ready samples\n", + "03. Training and evaluating models properly\n", + "04. Choosing a model: from logistic regression to XGBoost\n", + "05. Interpreting predictions\n", + "06. Medical codes: lookups, mappings and groupers\n", + "07. Clinical text: classification and medical coding\n", + "08. Contributing a dataset or task to PyHealth\n", + "\n", + "Questions or problems? Open an issue at https://github.com/sunlabuiuc/PyHealth/issues. If PyHealth is useful to you, a star on GitHub helps the project." + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/tutorials/series/05_interpreting_predictions.ipynb b/examples/tutorials/series/05_interpreting_predictions.ipynb new file mode 100644 index 000000000..fb42e0506 --- /dev/null +++ b/examples/tutorials/series/05_interpreting_predictions.ipynb @@ -0,0 +1,452 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-00", + "metadata": {}, + "source": [ + "# PyHealth Tutorial 05: Interpreting predictions\n", + "\n", + "**What you will learn**\n", + "- Which inputs drive a model overall (global importance) and for one patient (local explanation)\n", + "- Exact TreeSHAP for XGBoost, and Integrated Gradients for neural models\n", + "- How to check whether an explanation is faithful to the model, not just plausible\n", + "\n", + "**Time:** about 20 minutes on a free Colab CPU runtime. \n", + "**Before this:** Tutorial 04 (we reuse its heart-failure task).\n", + "\n", + "All data used here is synthetic or public, so every cell runs without credentials." + ] + }, + { + "cell_type": "markdown", + "id": "cell-01", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub. Once 2.1 is on PyPI this becomes `pip install \"pyhealth>=2.1\"`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-02", + "metadata": { + "tags": [ + "install" + ] + }, + "outputs": [], + "source": [ + "%pip install -q \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"" + ] + }, + { + "cell_type": "markdown", + "id": "cell-03", + "metadata": {}, + "source": [ + "Clinicians reasonably ask *why* a model flags a patient. PyHealth has\n", + "two families of tools:\n", + "\n", + "- **Exact Shapley values for tree models** (`XGBoostModel.explain`,\n", + " `TreeSHAP`): fast and deterministic, computed by XGBoost itself.\n", + "- **Attribution methods for neural models** in `pyhealth.interpret`:\n", + " Integrated Gradients, DeepLIFT, GIM, attention-based methods (Chefer,\n", + " attention rollout), and model-agnostic SHAP and LIME. They share one\n", + " interface: `attribute(**batch)` returns a score per input position,\n", + " keyed by feature name.\n", + "\n", + "We use the heart-failure phenotyping task from Tutorial 04: does an\n", + "admission include a heart-failure diagnosis, judged from its drugs and\n", + "procedures?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-04", + "metadata": {}, + "outputs": [], + "source": [ + "from collections import defaultdict\n", + "\n", + "from pyhealth.datasets import MIMIC3Dataset, PatientSplit, get_dataloader\n", + "from pyhealth.tasks import BaseTask\n", + "\n", + "\n", + "class HeartFailureFromMedications(BaseTask):\n", + " \"\"\"Does an admission include a heart-failure diagnosis (ICD-9 428.x)?\"\"\"\n", + "\n", + " task_name = \"HeartFailureFromMedications\"\n", + " input_schema = {\"drugs\": \"sequence\", \"procedures\": \"sequence\"}\n", + " output_schema = {\"heart_failure\": \"binary\"}\n", + "\n", + " def __call__(self, patient):\n", + " # Fetch each event type once, then group by admission: much\n", + " # faster than one filtered query per admission.\n", + " by_admission = defaultdict(lambda: {\"drugs\": [], \"diagnoses\": [], \"procedures\": []})\n", + " for e in patient.get_events(event_type=\"prescriptions\"):\n", + " if e.drug:\n", + " by_admission[e.hadm_id][\"drugs\"].append(e.drug)\n", + " for e in patient.get_events(event_type=\"diagnoses_icd\"):\n", + " by_admission[e.hadm_id][\"diagnoses\"].append(e.icd9_code)\n", + " for e in patient.get_events(event_type=\"procedures_icd\"):\n", + " by_admission[e.hadm_id][\"procedures\"].append(e.icd9_code)\n", + "\n", + " samples = []\n", + " for hadm_id, codes in by_admission.items():\n", + " if not codes[\"drugs\"] or not codes[\"diagnoses\"]:\n", + " continue\n", + " samples.append({\n", + " \"patient_id\": patient.patient_id,\n", + " \"visit_id\": hadm_id,\n", + " \"drugs\": codes[\"drugs\"],\n", + " \"procedures\": codes[\"procedures\"] or [\"none\"],\n", + " \"heart_failure\": int(any(c.startswith(\"428\") for c in codes[\"diagnoses\"])),\n", + " })\n", + " return samples\n", + "\n", + "\n", + "dataset = MIMIC3Dataset(\n", + " root=\"https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III\",\n", + " tables=[\"diagnoses_icd\", \"procedures_icd\", \"prescriptions\"],\n", + " cache_dir=\"pyhealth_cache\",\n", + ")\n", + "train_ds, val_ds, test_ds = dataset.set_task(\n", + " HeartFailureFromMedications(), split=PatientSplit((0.7, 0.1, 0.2), seed=7)\n", + ")\n", + "val_loader = get_dataloader(val_ds, batch_size=512)\n", + "test_loader = get_dataloader(test_ds, batch_size=512)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-05", + "metadata": {}, + "source": [ + "## 1. Tree models: exact Shapley values\n", + "\n", + "A Shapley value says how much one input moved a prediction away from\n", + "the average prediction. For tree ensembles they can be computed\n", + "exactly. They are on the **log-odds** scale and add up: the bias plus\n", + "all contributions equals the model's raw score for that sample.\n", + "\n", + "First, fit the model (as in Tutorial 04):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-06", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.models import XGBoostModel\n", + "from pyhealth.trainer import Trainer\n", + "\n", + "xgb = XGBoostModel(train_ds, bag_of_codes=True, n_estimators=400, max_depth=4,\n", + " learning_rate=0.05, subsample=0.8, colsample_bytree=0.8,\n", + " random_state=0, early_stopping_rounds=50, eval_metric=\"aucpr\")\n", + "xgb.fit(get_dataloader(train_ds, batch_size=1024), val_loader)\n", + "print(Trainer(model=xgb, metrics=[\"pr_auc\", \"roc_auc\"], enable_logging=False).evaluate(test_loader))" + ] + }, + { + "cell_type": "markdown", + "id": "cell-07", + "metadata": {}, + "source": [ + "### Global importance\n", + "\n", + "`mean_abs_shap` averages the size of each column's contribution over a\n", + "dataset: which inputs the model leans on most, in either direction.\n", + "Columns are named `field=code`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-08", + "metadata": {}, + "outputs": [], + "source": [ + "ranking = xgb.mean_abs_shap(test_loader)\n", + "for name, value in list(ranking.items())[:12]:\n", + " print(f\"{value:.3f} {name}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-09", + "metadata": {}, + "source": [ + "Do these make clinical sense? Diuretics (furosemide), beta-blockers,\n", + "ACE inhibitors and digoxin are standard heart-failure drugs, so we\n", + "would expect them near the top. A ranking that does not make sense is\n", + "worth investigating: it can reveal leakage or a data problem rather\n", + "than medical insight. (The synthetic data only partly preserves real\n", + "prescribing patterns.)\n", + "\n", + "### Explaining one admission\n", + "\n", + "`explain` gives every column's contribution for each sample. Let's take\n", + "the test admission the model is most confident about and list what\n", + "pushed its score up or down." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-10", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "\n", + "batch = next(iter(test_loader))\n", + "out = xgb.explain(**batch)\n", + "names = [n for field in xgb.feature_layout for n in out[\"feature_names\"][field[\"key\"]]]\n", + "contrib = torch.cat([out[\"attributions\"][field[\"key\"]] for field in xgb.feature_layout], dim=1)\n", + "\n", + "i = int(out[\"logit\"][:, 0].argmax())\n", + "prob = torch.sigmoid(out[\"logit\"][i, 0])\n", + "print(f\"admission {batch['visit_id'][i]}: predicted heart-failure probability {prob:.2f}, \"\n", + " f\"true label {int(batch['heart_failure'][i])}\")\n", + "order = contrib[i].argsort(descending=True)\n", + "print(\"\\npushes the score up:\")\n", + "for j in order[:6]:\n", + " print(f\" {contrib[i, j]:+.3f} {names[j]}\")\n", + "print(\"pushes the score down:\")\n", + "for j in order[-3:]:\n", + " print(f\" {contrib[i, j]:+.3f} {names[j]}\")\n", + "total = out[\"bias\"][i, 0] + contrib[i].sum() if out[\"bias\"].ndim > 1 else out[\"bias\"][i] + contrib[i].sum()\n", + "print(f\"\\nbias + contributions = {float(total):.3f}, model log-odds = {float(out['logit'][i, 0]):.3f}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-11", + "metadata": {}, + "source": [ + "The last line checks the bookkeeping: contributions plus bias equal the\n", + "model output exactly. Note that an *absent* drug can also contribute\n", + "(its absence lowered or raised the score), which is why some names\n", + "above may not appear in this admission's list.\n", + "\n", + "`pyhealth.interpret.methods.TreeSHAP` returns the same values in the\n", + "shape of each input field, like the neural interpreters below.\n", + "\n", + "## 2. Neural models: Integrated Gradients\n", + "\n", + "For a neural model, Integrated Gradients compares the input with a\n", + "neutral baseline (here, every code replaced by the unknown token) and\n", + "accumulates gradients along the path between them. The result is one\n", + "score per input position: per drug in the list, per procedure." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-12", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.models import MLP\n", + "\n", + "torch.manual_seed(0)\n", + "mlp = MLP(dataset=train_ds)\n", + "trainer = Trainer(model=mlp, metrics=[\"pr_auc\", \"roc_auc\"], enable_logging=False)\n", + "trainer.train(\n", + " train_dataloader=get_dataloader(train_ds, batch_size=256, shuffle=True),\n", + " val_dataloader=val_loader,\n", + " epochs=5,\n", + " monitor=\"pr_auc\",\n", + ")\n", + "print(trainer.evaluate(test_loader))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-13", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.interpret.methods import IntegratedGradients\n", + "\n", + "ig = IntegratedGradients(mlp, steps=32)\n", + "small = next(iter(get_dataloader(test_ds, batch_size=16)))\n", + "attributions = ig.attribute(**small)\n", + "print({k: tuple(v.shape) for k, v in attributions.items()})" + ] + }, + { + "cell_type": "markdown", + "id": "cell-14", + "metadata": {}, + "source": [ + "Each attribution tensor has the shape of its input: one score per drug\n", + "position. To read them, map the token ids back to drug names with the\n", + "processor's vocabulary (Tutorial 02):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-15", + "metadata": {}, + "outputs": [], + "source": [ + "vocab = train_ds.input_processors[\"drugs\"].code_vocab\n", + "id_to_drug = {i: d for d, i in vocab.items()}\n", + "\n", + "with torch.no_grad():\n", + " probs = mlp(**small)[\"y_prob\"][:, 0]\n", + "k = int(probs.argmax())\n", + "print(f\"admission {small['visit_id'][k]}: probability {probs[k]:.2f}, true label {int(small['heart_failure'][k])}\")\n", + "scores = attributions[\"drugs\"][k].detach()\n", + "tokens = small[\"drugs\"][k]\n", + "ranked = sorted(((float(s), id_to_drug[int(t)]) for s, t in zip(scores, tokens) if int(t) > 1), reverse=True)\n", + "for s, name in ranked[:8]:\n", + " print(f\" {s:+.4f} {name}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-16", + "metadata": {}, + "source": [ + "## 3. Is the explanation faithful?\n", + "\n", + "A plausible-looking explanation is not necessarily the one the model\n", + "uses. A standard check is a **deletion test**: remove the inputs the\n", + "explanation ranks highest and see how far the prediction falls. If the\n", + "explanation is faithful, removing its top drugs should lower the\n", + "predicted risk much more than removing the same number of random\n", + "drugs. (This is the idea behind the *comprehensiveness* metric.)\n", + "\n", + "We remove a drug by replacing its id with the padding id 0, which the\n", + "model ignores." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-17", + "metadata": {}, + "outputs": [], + "source": [ + "def remove_drugs(batch, scores, k, generator=None):\n", + " \"\"\"Copy of the batch with each sample's k highest-scoring drugs set to padding (always keeps one).\"\"\"\n", + " drugs = batch[\"drugs\"].clone()\n", + " real = drugs > 1 # skip padding (0) and unknown (1)\n", + " if generator is not None: # random ranking instead\n", + " scores = torch.rand(drugs.shape, generator=generator)\n", + " ranked = scores.masked_fill(~real, float(\"-inf\")).argsort(dim=1, descending=True)\n", + " for row in range(drugs.shape[0]):\n", + " top = ranked[row, : min(k, int(real[row].sum()) - 1)] # keep at least one drug\n", + " drugs[row, top] = 0\n", + " return {**batch, \"drugs\": drugs}\n", + "\n", + "\n", + "def risk(batch):\n", + " with torch.no_grad():\n", + " return mlp(**batch)[\"y_prob\"][:, 0]\n", + "\n", + "\n", + "eval_batch = next(iter(get_dataloader(test_ds, batch_size=1024)))\n", + "ig_scores = ig.attribute(**eval_batch)[\"drugs\"].detach()\n", + "base = risk(eval_batch)\n", + "flagged = base > 0.5 # admissions the model calls positive\n", + "gen = torch.Generator().manual_seed(0)\n", + "print(f\"{int(flagged.sum())} admissions predicted positive; mean risk {base[flagged].mean():.3f}\\n\")\n", + "print(\"drugs removed drop with IG ranking drop with random ranking\")\n", + "for k in [1, 3, 5]:\n", + " ig_drop = (base - risk(remove_drugs(eval_batch, ig_scores, k)))[flagged].mean()\n", + " rnd_drop = (base - risk(remove_drugs(eval_batch, ig_scores, k, generator=gen)))[flagged].mean()\n", + " print(f\"{k:13d} {ig_drop:20.3f} {rnd_drop:24.3f}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-18", + "metadata": {}, + "source": [ + "If the Integrated Gradients column falls clearly faster than the random\n", + "column, the attributions point at inputs the model really relies on.\n", + "`pyhealth.metrics.interpretability` packages this idea as\n", + "comprehensiveness and sufficiency scores for whole datasets." + ] + }, + { + "cell_type": "markdown", + "id": "cell-19", + "metadata": {}, + "source": [ + "## Choosing a method\n", + "\n", + "| Situation | Use |\n", + "|---|---|\n", + "| XGBoost or another tree model | `xgb.explain` / `mean_abs_shap` / `TreeSHAP`: exact and fast |\n", + "| Neural model on codes | `IntegratedGradients`, `DeepLift`, `GIM` |\n", + "| Transformer, want attention-based relevance | `CheferRelevance`, `AttentionRollout` |\n", + "| Any model, treat it as a black box | `ShapExplainer`, `LimeExplainer` (slower, sampling-based) |\n", + "\n", + "Whatever you use, check faithfulness as above, and treat explanations\n", + "as hypotheses to examine with clinicians, not as causal findings: they\n", + "describe the model, and the model reflects the data, biases included.\n", + "\n", + "## Summary\n", + "\n", + "- Tree models give exact Shapley values; they add up to the model's\n", + " log-odds output.\n", + "- Neural interpreters share `attribute(**batch)` and return one score\n", + " per input position; map ids back with the processor vocabulary.\n", + "- Measure faithfulness (comprehensiveness, sufficiency) against a\n", + " random baseline before trusting an explanation." + ] + }, + { + "cell_type": "markdown", + "id": "cell-20", + "metadata": {}, + "source": [ + "## Where to go next\n", + "\n", + "**Next:** Tutorial 06, *Medical codes: lookups, mappings and groupers*.\n", + "\n", + "The whole series:\n", + "\n", + "00. Quickstart: your first clinical prediction model\n", + "01. Datasets: patients, events and your own data\n", + "02. Tasks and processors: from patients to model-ready samples\n", + "03. Training and evaluating models properly\n", + "04. Choosing a model: from logistic regression to XGBoost\n", + "05. Interpreting predictions\n", + "06. Medical codes: lookups, mappings and groupers\n", + "07. Clinical text: classification and medical coding\n", + "08. Contributing a dataset or task to PyHealth\n", + "\n", + "Questions or problems? Open an issue at https://github.com/sunlabuiuc/PyHealth/issues. If PyHealth is useful to you, a star on GitHub helps the project." + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/tutorials/series/06_medical_codes.ipynb b/examples/tutorials/series/06_medical_codes.ipynb new file mode 100644 index 000000000..1462ce3f1 --- /dev/null +++ b/examples/tutorials/series/06_medical_codes.ipynb @@ -0,0 +1,343 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-00", + "metadata": {}, + "source": [ + "# PyHealth Tutorial 06: Medical codes: lookups, mappings and groupers\n", + "\n", + "**What you will learn**\n", + "- What the common code systems are (ICD-9, ICD-10, CCS, ATC, NDC) and how they relate\n", + "- How to look up a code's name, parents and children\n", + "- How to translate between ICD-9 and ICD-10 and group codes into broader categories\n", + "- How to use a grouping inside a task to shrink the vocabulary a model must learn\n", + "\n", + "**Time:** about 15 minutes on a free Colab CPU runtime. \n", + "**Before this:** Tutorial 02 (processors and vocabularies).\n", + "\n", + "All data used here is synthetic or public, so every cell runs without credentials." + ] + }, + { + "cell_type": "markdown", + "id": "cell-01", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub. Once 2.1 is on PyPI this becomes `pip install \"pyhealth>=2.1\"`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-02", + "metadata": { + "tags": [ + "install" + ] + }, + "outputs": [], + "source": [ + "%pip install -q \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"" + ] + }, + { + "cell_type": "markdown", + "id": "cell-03", + "metadata": {}, + "source": [ + "## Why codes need care\n", + "\n", + "EHR data records diagnoses, procedures and drugs as codes from\n", + "standard systems. Some things to know:\n", + "\n", + "| System | Codes | Example |\n", + "|---|---|---|\n", + "| ICD-9-CM / ICD-10-CM | diagnoses (US; ICD-10 since 2015) | `428.0` / `I50.9` heart failure |\n", + "| ICD-9-PROC / ICD-10-PCS | procedures | `39.95` hemodialysis |\n", + "| CCS, CCSR | groups of related ICD codes (a few hundred categories) | CCS 108: congestive heart failure |\n", + "| NDC | US drug products (package level) | `00054429731` |\n", + "| ATC | drugs by anatomy and chemistry, 5 levels | `C03CA01` furosemide |\n", + "\n", + "Fine-grained codes give large, sparse vocabularies: thousands of codes,\n", + "many seen only a handful of times. Grouping them into broader\n", + "categories often helps models, and translating between versions lets\n", + "you combine data from before and after the ICD-10 switch.\n", + "\n", + "`pyhealth.medcode` has two tools:\n", + "- `InnerMap`: everything *within* one system (names, hierarchy).\n", + "- `CrossMap`: mappings *between* systems." + ] + }, + { + "cell_type": "markdown", + "id": "cell-04", + "metadata": {}, + "source": [ + "## Looking codes up: `InnerMap`\n", + "\n", + "The first load of a vocabulary downloads a small table and caches it.\n", + "MIMIC stores ICD-9 codes without the dot (`4280`); lookups accept both\n", + "forms." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-05", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.medcode import InnerMap\n", + "\n", + "icd9 = InnerMap.load(\"ICD9CM\")\n", + "for c in [\"4280\", \"428.0\", \"25000\", \"486\"]:\n", + " print(f\"{c:6s} {icd9.lookup(c)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-06", + "metadata": {}, + "source": [ + "Codes form a hierarchy. Ancestors are the broader categories a code\n", + "belongs to; descendants are its more specific codes:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-07", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"ancestors of 428.0:\")\n", + "for a in icd9.get_ancestors(\"428.0\"):\n", + " print(f\" {a:8s} {icd9.lookup(a)}\")\n", + "print(\"\\nmore specific codes under 428.2 (systolic heart failure):\")\n", + "for d in icd9.get_descendants(\"428.2\"):\n", + " print(f\" {d:8s} {icd9.lookup(d)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-08", + "metadata": {}, + "outputs": [], + "source": [ + "atc = InnerMap.load(\"ATC\")\n", + "for c in [\"C03CA01\", \"C03CA\", \"C03\", \"C\"]:\n", + " print(f\"{c:8s} {atc.lookup(c)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-09", + "metadata": {}, + "source": [ + "## Translating and grouping: `CrossMap`\n", + "\n", + "`CrossMap.load(source, target)` maps codes from one system to another.\n", + "`map` returns a list, because one code can map to several." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-10", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.medcode import CrossMap\n", + "\n", + "icd9_to_icd10 = CrossMap.load(\"ICD9CM\", \"ICD10CM\")\n", + "icd10_to_icd9 = CrossMap.load(\"ICD10CM\", \"ICD9CM\")\n", + "for c in [\"428.0\", \"250.00\", \"486\", \"038.9\"]:\n", + " print(f\"ICD-9 {c:7s} -> ICD-10 {icd9_to_icd10.map(c)}\")\n", + "print()\n", + "for c in [\"I50.9\", \"E11.9\", \"A41.9\"]:\n", + " print(f\"ICD-10 {c:6s} -> ICD-9 {icd10_to_icd9.map(c)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-11", + "metadata": {}, + "source": [ + "Notice `038.9` (septicemia) maps to `A41.9`, which maps back to\n", + "`995.91`, not `038.9`. Translation is not a round trip, so translate\n", + "once, in one direction, and say which in your methods.\n", + "\n", + "Groupers collapse many codes into clinically meaningful categories:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-12", + "metadata": {}, + "outputs": [], + "source": [ + "to_ccs = CrossMap.load(\"ICD9CM\", \"CCSCM\")\n", + "ccs = InnerMap.load(\"CCSCM\")\n", + "for c in [\"428.0\", \"428.22\", \"402.91\", \"250.00\", \"250.60\"]:\n", + " group = to_ccs.map(c)\n", + " print(f\"{c:7s} {icd9.lookup(c)[:45]:45s} -> CCS {group} {ccs.lookup(group[0]) if group else ''}\")\n", + "\n", + "print()\n", + "to_chapter = CrossMap.load(\"ICD10CM\", \"ICD10CHAPTER\")\n", + "to_ccsr = CrossMap.load(\"ICD10CM\", \"CCSR\")\n", + "for c in [\"I50.9\", \"E11.9\", \"J18.9\"]:\n", + " print(f\"{c:6s} chapter {to_chapter.map(c)} CCSR {to_ccsr.map(c)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-13", + "metadata": {}, + "source": [ + "Drugs work the same way: NDC product codes map to ATC classes, and\n", + "`target_kwargs={\"level\": n}` picks how coarse (level 3 is the\n", + "therapeutic subgroup, level 5 the single substance):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-14", + "metadata": {}, + "outputs": [], + "source": [ + "ndc_to_atc = CrossMap.load(\"NDC\", \"ATC\")\n", + "ndc = \"00054429731\"\n", + "for level in [3, 4, 5]:\n", + " codes = ndc_to_atc.map(ndc, target_kwargs={\"level\": level})\n", + " print(f\"level {level}: {[(c, atc.lookup(c)) for c in codes]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-15", + "metadata": {}, + "source": [ + "## Using a grouping in a task\n", + "\n", + "Every task accepts `code_mapping={field: (source, target)}`. The\n", + "processors then map each raw code before building the vocabulary, so\n", + "the model sees CCS categories and ATC classes instead of raw codes.\n", + "Let's compare the vocabulary sizes on synthetic MIMIC-III:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-16", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.datasets import MIMIC3Dataset\n", + "from pyhealth.tasks import MortalityPredictionMIMIC3\n", + "\n", + "dataset = MIMIC3Dataset(\n", + " root=\"https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III\",\n", + " tables=[\"diagnoses_icd\", \"procedures_icd\", \"prescriptions\"],\n", + " cache_dir=\"pyhealth_cache\",\n", + ")\n", + "raw = dataset.set_task(MortalityPredictionMIMIC3())\n", + "grouped = dataset.set_task(MortalityPredictionMIMIC3(code_mapping={\n", + " \"conditions\": (\"ICD9CM\", \"CCSCM\"),\n", + " \"procedures\": (\"ICD9PROC\", \"CCSPROC\"),\n", + " \"drugs\": (\"NDC\", \"ATC\"),\n", + "}))\n", + "for field in [\"conditions\", \"procedures\", \"drugs\"]:\n", + " n_raw = len(raw.input_processors[field].code_vocab)\n", + " n_grouped = len(grouped.input_processors[field].code_vocab)\n", + " print(f\"{field:11s} {n_raw:5d} raw codes -> {n_grouped:4d} groups\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-17", + "metadata": {}, + "source": [ + "A model on the grouped version has far fewer embeddings to learn,\n", + "each seen many more times. Whether that helps depends on the question:\n", + "groupings lose detail (all heart-failure subtypes become one category),\n", + "so try both and compare on validation data (Tutorial 03).\n", + "\n", + "Codes a mapping does not cover fall back to the raw code, so nothing\n", + "is silently dropped. A mapping object records what it could not map:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-18", + "metadata": {}, + "outputs": [], + "source": [ + "translator = CrossMap.load(\"ICD9CM\", \"ICD10CM\")\n", + "for c in [\"428.0\", \"999.99\"]:\n", + " print(c, \"->\", translator.map(c))\n", + "print(\"unmapped so far:\", translator.unmapped_codes)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-19", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "- `InnerMap.load(system)`: `lookup`, `get_ancestors`, `get_descendants`.\n", + "- `CrossMap.load(source, target).map(code)`: translations and\n", + " groupers; translation is not a round trip.\n", + "- `code_mapping=` on any task groups codes before the vocabulary is\n", + " built, shrinking what the model must learn." + ] + }, + { + "cell_type": "markdown", + "id": "cell-20", + "metadata": {}, + "source": [ + "## Where to go next\n", + "\n", + "**Next:** Tutorial 07, *Clinical text: classification and medical coding*.\n", + "\n", + "The whole series:\n", + "\n", + "00. Quickstart: your first clinical prediction model\n", + "01. Datasets: patients, events and your own data\n", + "02. Tasks and processors: from patients to model-ready samples\n", + "03. Training and evaluating models properly\n", + "04. Choosing a model: from logistic regression to XGBoost\n", + "05. Interpreting predictions\n", + "06. Medical codes: lookups, mappings and groupers\n", + "07. Clinical text: classification and medical coding\n", + "08. Contributing a dataset or task to PyHealth\n", + "\n", + "Questions or problems? Open an issue at https://github.com/sunlabuiuc/PyHealth/issues. If PyHealth is useful to you, a star on GitHub helps the project." + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/tutorials/series/07_clinical_text.ipynb b/examples/tutorials/series/07_clinical_text.ipynb new file mode 100644 index 000000000..943eff0e4 --- /dev/null +++ b/examples/tutorials/series/07_clinical_text.ipynb @@ -0,0 +1,359 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-00", + "metadata": {}, + "source": [ + "# PyHealth Tutorial 07: Clinical text: classification and medical coding\n", + "\n", + "**What you will learn**\n", + "- How to fine-tune a Hugging Face language model on clinical notes with PyHealth\n", + "- Multiclass text classification: which specialty wrote this report?\n", + "- Multilabel medical coding: which ICD codes does this note support?\n", + "- Why a simple TF-IDF baseline belongs next to every language model\n", + "\n", + "**Time:** about 25 minutes on a free Colab CPU runtime. \n", + "**Before this:** Tutorials 00 and 02. A GPU runtime (Runtime > Change runtime type > T4 GPU) makes training much faster.\n", + "\n", + "All data used here is synthetic or public, so every cell runs without credentials." + ] + }, + { + "cell_type": "markdown", + "id": "cell-01", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub. Once 2.1 is on PyPI this becomes `pip install \"pyhealth>=2.1\"`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-02", + "metadata": { + "tags": [ + "install" + ] + }, + "outputs": [], + "source": [ + "%pip install -q \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"" + ] + }, + { + "cell_type": "markdown", + "id": "cell-03", + "metadata": {}, + "source": [ + "## Text is just another field\n", + "\n", + "For PyHealth, a note is one more input field. The `text` processor\n", + "keeps it as a string, and `TransformersModel` wraps any Hugging Face\n", + "encoder: it tokenizes the text (first 256 tokens), encodes it, and\n", + "adds a prediction head that matches the task's label (multiclass,\n", + "multilabel, binary).\n", + "\n", + "Model size matters on a free runtime. We pick automatically: a\n", + "clinical BERT (110M parameters, trained on MIMIC notes) on a GPU, or a\n", + "small general BERT (11M parameters) on a CPU." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-04", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "\n", + "if torch.cuda.is_available():\n", + " MODEL_NAME, EPOCHS = \"emilyalsentzer/Bio_ClinicalBERT\", 3\n", + "else:\n", + " MODEL_NAME, EPOCHS = \"google/bert_uncased_L-4_H-256_A-4\", 2\n", + "print(\"device:\", \"cuda\" if torch.cuda.is_available() else \"cpu\", \"| model:\", MODEL_NAME)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-05", + "metadata": {}, + "source": [ + "## Part A. Which specialty wrote this report?\n", + "\n", + "The [MTSamples](https://www.kaggle.com/datasets/tboyle10/medicaltranscriptions)\n", + "collection has about 5,000 transcribed medical reports, each labelled\n", + "with a medical specialty. These are sample documents, not patient\n", + "records, so each report is its own \"patient\"." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-06", + "metadata": {}, + "outputs": [], + "source": [ + "import urllib.request\n", + "import zipfile\n", + "\n", + "urllib.request.urlretrieve(\n", + " \"https://storage.googleapis.com/pyhealth/medical_transcriptions_data/MedicalTranscriptions.zip\",\n", + " \"MedicalTranscriptions.zip\",\n", + ")\n", + "zipfile.ZipFile(\"MedicalTranscriptions.zip\").extractall(\".\")\n", + "\n", + "from pyhealth.datasets import MedicalTranscriptionsDataset\n", + "\n", + "mts = MedicalTranscriptionsDataset(root=\"MedicalTranscriptions\", cache_dir=\"pyhealth_cache\")\n", + "mts.stats()" + ] + }, + { + "cell_type": "markdown", + "id": "cell-07", + "metadata": {}, + "source": [ + "The dataset comes with a default task, text in and specialty out:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-08", + "metadata": {}, + "outputs": [], + "source": [ + "print(mts.default_task.input_schema, \"->\", mts.default_task.output_schema)\n", + "reports = mts.set_task()\n", + "\n", + "sample = reports[0]\n", + "print(f\"{len(reports)} reports\")\n", + "print(\"text:\", sample[\"transcription\"][:300], \"...\")\n", + "labels = reports.output_processors[\"medical_specialty\"].label_vocab\n", + "print(f\"\\n{len(labels)} specialties, e.g. {list(labels)[:6]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-09", + "metadata": {}, + "source": [ + "Each report is its own patient, so a sample-level split is fine here.\n", + "Then the usual PyHealth steps: model, trainer, evaluation. For\n", + "multiclass labels the trainer reports accuracy and F1; macro F1\n", + "weights every specialty equally, so rare specialties count as much as\n", + "common ones.\n", + "\n", + "Loading the model prints a table of \"unexpected\" weights. That is\n", + "normal: the pretraining heads of the checkpoint are dropped, and\n", + "PyHealth adds a new classification head for our labels." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-10", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.datasets import get_dataloader, split_by_sample\n", + "from pyhealth.models import TransformersModel\n", + "from pyhealth.trainer import Trainer\n", + "\n", + "train_ds, val_ds, test_ds = split_by_sample(reports, [0.7, 0.1, 0.2], seed=0)\n", + "model = TransformersModel(dataset=reports, model_name=MODEL_NAME)\n", + "trainer = Trainer(model=model, metrics=[\"accuracy\", \"f1_macro\", \"f1_weighted\"])\n", + "trainer.train(\n", + " train_dataloader=get_dataloader(train_ds, batch_size=16, shuffle=True),\n", + " val_dataloader=get_dataloader(val_ds, batch_size=32),\n", + " epochs=EPOCHS,\n", + " monitor=\"f1_weighted\",\n", + " optimizer_params={\"lr\": 5e-5},\n", + ")\n", + "bert_scores = trainer.evaluate(get_dataloader(test_ds, batch_size=32))\n", + "print(bert_scores)" + ] + }, + { + "cell_type": "markdown", + "id": "cell-11", + "metadata": {}, + "source": [ + "### A baseline to beat\n", + "\n", + "Bag-of-words models are fast and often strong on text classification.\n", + "A TF-IDF + logistic regression baseline on the same split takes\n", + "seconds; a language model should clearly beat it to justify its cost." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-12", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.feature_extraction.text import TfidfVectorizer\n", + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.metrics import accuracy_score, f1_score\n", + "\n", + "def texts_and_labels(ds):\n", + " return [ds[i][\"transcription\"] for i in range(len(ds))], [int(ds[i][\"medical_specialty\"]) for i in range(len(ds))]\n", + "\n", + "x_train, y_train = texts_and_labels(train_ds)\n", + "x_test, y_test = texts_and_labels(test_ds)\n", + "tfidf = TfidfVectorizer(min_df=2, ngram_range=(1, 2), sublinear_tf=True)\n", + "clf = LogisticRegression(max_iter=2000).fit(tfidf.fit_transform(x_train), y_train)\n", + "pred = clf.predict(tfidf.transform(x_test))\n", + "print(f\"TF-IDF + LR accuracy {accuracy_score(y_test, pred):.3f} \"\n", + " f\"F1 macro {f1_score(y_test, pred, average='macro'):.3f} \"\n", + " f\"F1 weighted {f1_score(y_test, pred, average='weighted'):.3f}\")\n", + "print(f\"{MODEL_NAME.split('/')[-1]:13s} accuracy {bert_scores['accuracy']:.3f} \"\n", + " f\"F1 macro {bert_scores['f1_macro']:.3f} F1 weighted {bert_scores['f1_weighted']:.3f}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-13", + "metadata": {}, + "source": [ + "Scores on this dataset are modest for any model: several \"specialties\"\n", + "(Surgery, Consult, Discharge Summary) describe the *kind* of document\n", + "rather than the field, and overlap heavily with the others. Looking at\n", + "the labels before modelling saves a lot of tuning.\n", + "\n", + "## Part B. Medical coding from notes\n", + "\n", + "Medical coding assigns ICD codes to a hospital stay from its\n", + "documentation. It is a **multilabel** problem: each note supports\n", + "several codes at once. PyHealth's `MIMIC3ICD9Coding` task joins a\n", + "patient's notes into one text and collects the diagnosis and\n", + "procedure codes as labels:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-14", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.datasets import MIMIC3Dataset\n", + "from pyhealth.tasks import MIMIC3ICD9Coding\n", + "\n", + "mimic = MIMIC3Dataset(\n", + " root=\"https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III\",\n", + " tables=[\"diagnoses_icd\", \"procedures_icd\", \"noteevents\"],\n", + " cache_dir=\"pyhealth_cache\",\n", + ")\n", + "coding_task = MIMIC3ICD9Coding()\n", + "print(coding_task.input_schema, \"->\", coding_task.output_schema)\n", + "notes = mimic.set_task(coding_task)\n", + "codes = notes.output_processors[\"icd_codes\"].label_vocab\n", + "print(f\"{len(notes)} samples, {len(codes)} distinct codes\")\n", + "print(\"first sample has\", int(notes[0][\"icd_codes\"].sum()), \"codes\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-15", + "metadata": {}, + "source": [ + "With thousands of possible codes, most appearing only a few times,\n", + "coding is hard. Two habits from practice:\n", + "\n", + "- Report **micro F1** (dominated by frequent codes) and **macro F1**\n", + " (every code equal), plus ranking metrics such as precision@k.\n", + "- Restrict to the most frequent codes (for example the top 50) for a\n", + " first model, as most published MIMIC coding benchmarks do.\n", + "\n", + "The training code is identical to Part A; only the task changes. (On\n", + "synthetic notes the text carries little information about the codes,\n", + "so expect low scores; the workflow is what transfers to real MIMIC.)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-16", + "metadata": {}, + "outputs": [], + "source": [ + "from pyhealth.datasets import split_by_patient\n", + "\n", + "train_n, val_n, test_n = split_by_patient(notes, [0.7, 0.1, 0.2], seed=0)\n", + "coder = TransformersModel(dataset=notes, model_name=MODEL_NAME)\n", + "coding_trainer = Trainer(model=coder, metrics=[\"f1_micro\", \"f1_macro\", \"pr_auc_samples\"])\n", + "coding_trainer.train(\n", + " train_dataloader=get_dataloader(train_n, batch_size=16, shuffle=True),\n", + " val_dataloader=get_dataloader(val_n, batch_size=32),\n", + " epochs=1,\n", + " monitor=\"pr_auc_samples\",\n", + " optimizer_params={\"lr\": 5e-5},\n", + ")\n", + "print(coding_trainer.evaluate(get_dataloader(test_n, batch_size=32)))" + ] + }, + { + "cell_type": "markdown", + "id": "cell-17", + "metadata": {}, + "source": [ + "## Summary\n", + "\n", + "- A `text` field plus `TransformersModel(model_name=...)` fine-tunes\n", + " any Hugging Face encoder; the task decides multiclass vs multilabel.\n", + "- Pick the model size for your hardware; use a GPU runtime for\n", + " full-size clinical models.\n", + "- Always compare with a TF-IDF baseline, and inspect the labels before\n", + " tuning.\n", + "- Medical coding is multilabel and long-tailed: report micro and macro\n", + " metrics, and start with frequent codes." + ] + }, + { + "cell_type": "markdown", + "id": "cell-18", + "metadata": {}, + "source": [ + "## Where to go next\n", + "\n", + "**Next:** Tutorial 08, *Contributing a dataset or task to PyHealth*.\n", + "\n", + "The whole series:\n", + "\n", + "00. Quickstart: your first clinical prediction model\n", + "01. Datasets: patients, events and your own data\n", + "02. Tasks and processors: from patients to model-ready samples\n", + "03. Training and evaluating models properly\n", + "04. Choosing a model: from logistic regression to XGBoost\n", + "05. Interpreting predictions\n", + "06. Medical codes: lookups, mappings and groupers\n", + "07. Clinical text: classification and medical coding\n", + "08. Contributing a dataset or task to PyHealth\n", + "\n", + "Questions or problems? Open an issue at https://github.com/sunlabuiuc/PyHealth/issues. If PyHealth is useful to you, a star on GitHub helps the project." + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/tutorials/series/08_contributing.ipynb b/examples/tutorials/series/08_contributing.ipynb new file mode 100644 index 000000000..41e62ab9f --- /dev/null +++ b/examples/tutorials/series/08_contributing.ipynb @@ -0,0 +1,361 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-00", + "metadata": {}, + "source": [ + "# PyHealth Tutorial 08: Contributing a dataset or task to PyHealth\n", + "\n", + "**What you will learn**\n", + "- Whether your idea is a dataset, a task, or both\n", + "- How a dataset class is built: config, default tables, per-table preprocessing, a default task\n", + "- How to handle data that is not a table (files on disk)\n", + "- How to test it with tiny synthetic data, and what a PR needs to pass review\n", + "\n", + "**Time:** about 25 minutes on a free Colab CPU runtime. \n", + "**Before this:** Tutorials 01 and 02.\n", + "\n", + "All data used here is synthetic or public, so every cell runs without credentials." + ] + }, + { + "cell_type": "markdown", + "id": "cell-01", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub. Once 2.1 is on PyPI this becomes `pip install \"pyhealth>=2.1\"`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-02", + "metadata": { + "tags": [ + "install" + ] + }, + "outputs": [], + "source": [ + "%pip install -q \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"" + ] + }, + { + "cell_type": "markdown", + "id": "cell-03", + "metadata": {}, + "source": [ + "## Dataset or task?\n", + "\n", + "| You have... | Contribute |\n", + "|---|---|\n", + "| A new data *source* (a database, a public collection) | a dataset in `pyhealth/datasets/` |\n", + "| A new *question* about a source PyHealth already loads | a task in `pyhealth/tasks/` |\n", + "| A new source that comes with its own benchmark question | both, with the task as the dataset's `default_task` |\n", + "\n", + "A common mistake is to write a new dataset class for a new MIMIC\n", + "question (for example, synthetic data generation from MIMIC-III). That\n", + "is a task: it reuses `MIMIC3Dataset` and adds only the logic of the\n", + "question.\n", + "\n", + "## Anatomy of a dataset class\n", + "\n", + "In Tutorial 01 we loaded CSVs with `BaseDataset` and a YAML config. A\n", + "contributed dataset wraps the same thing in a class, so users only\n", + "pass `root`. Here it is for the small clinic data, with every hook a\n", + "real dataset uses:\n", + "\n", + "1. A default config shipped with the class.\n", + "2. Default tables that are always loaded (like `patients` and\n", + " `admissions` in MIMIC-III).\n", + "3. `preprocess_
`: optional per-table cleaning, applied lazily\n", + " before events are built.\n", + "4. `default_task`: what `dataset.set_task()` runs with no argument." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-04", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "root = Path(\"my_clinic\")\n", + "root.mkdir(exist_ok=True)\n", + "(root / \"patients.csv\").write_text(\n", + " \"patient_id,birth_year,sex\\n\" \"p1,1956,F\\n\" \"p2,1971,M\\n\" \"p3,1990,F\\n\"\n", + ")\n", + "(root / \"visits.csv\").write_text(\n", + " \"patient_id,visit_time,diagnosis,systolic_bp\\n\"\n", + " \"p1,2023-01-05 09:30,i10,152\\n\"\n", + " \"p1,2023-06-12 14:00,E11.9,148\\n\"\n", + " \"p1,2024-02-01 10:15,I10,139\\n\"\n", + " \"p2,2023-03-20 08:45,J45.909,121\\n\"\n", + " \"p3,2023-11-02 16:20,I10,135\\n\"\n", + " \"p3,2024-04-18 11:05,n18.3,141\\n\"\n", + ")\n", + "config_path = root / \"my_clinic.yaml\"\n", + "config_path.write_text('''\n", + "version: \"1.0\"\n", + "tables:\n", + " patients:\n", + " file_path: \"patients.csv\"\n", + " patient_id: \"patient_id\"\n", + " timestamp: null\n", + " attributes: [\"birth_year\", \"sex\"]\n", + " visits:\n", + " file_path: \"visits.csv\"\n", + " patient_id: \"patient_id\"\n", + " timestamp: \"visit_time\"\n", + " timestamp_format: \"%Y-%m-%d %H:%M\"\n", + " attributes: [\"diagnosis\", \"systolic_bp\"]\n", + "''')" + ] + }, + { + "cell_type": "markdown", + "id": "cell-05", + "metadata": {}, + "source": [ + "Notice the raw data has inconsistent code case (`i10` vs `I10`). The\n", + "`preprocess_visits` hook fixes that for everyone who uses the dataset.\n", + "Hooks receive a lazy dataframe through\n", + "[narwhals](https://narwhals-dev.github.io/narwhals/), which has the\n", + "same expression syntax as polars; column names arrive lowercased." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-06", + "metadata": {}, + "outputs": [], + "source": [ + "import narwhals as nw\n", + "from pyhealth.datasets import BaseDataset\n", + "from pyhealth.tasks import BaseTask\n", + "\n", + "\n", + "class HighBloodPressureNextVisit(BaseTask):\n", + " \"\"\"Is systolic blood pressure >= 140 at the next visit?\"\"\"\n", + "\n", + " task_name = \"HighBloodPressureNextVisit\"\n", + " input_schema = {\"diagnoses\": \"sequence\", \"last_bp\": \"tensor\"}\n", + " output_schema = {\"high_bp_next\": \"binary\"}\n", + "\n", + " def __call__(self, patient):\n", + " visits = patient.get_events(event_type=\"visits\")\n", + " return [\n", + " {\n", + " \"patient_id\": patient.patient_id,\n", + " \"diagnoses\": [v.diagnosis for v in visits[: i + 1]],\n", + " \"last_bp\": [float(visits[i].systolic_bp)],\n", + " \"high_bp_next\": int(float(visits[i + 1].systolic_bp) >= 140),\n", + " }\n", + " for i in range(len(visits) - 1)\n", + " ]\n", + "\n", + "\n", + "class MyClinicDataset(BaseDataset):\n", + " \"\"\"Visits and demographics from My Clinic.\n", + "\n", + " Args:\n", + " root: Folder with patients.csv and visits.csv.\n", + " config_path: Optional config; defaults to the one shipped with the class.\n", + " **kwargs: Passed to BaseDataset (cache_dir, dev, num_workers, ...).\n", + "\n", + " Examples:\n", + " >>> dataset = MyClinicDataset(root=\"/path/to/my_clinic\") # doctest: +SKIP\n", + " >>> samples = dataset.set_task() # doctest: +SKIP\n", + " \"\"\"\n", + "\n", + " def __init__(self, root, config_path=None, **kwargs):\n", + " # In the package this is Path(__file__).parent / \"configs\" / \"my_clinic.yaml\".\n", + " config_path = config_path or str(Path(root) / \"my_clinic.yaml\")\n", + " super().__init__(\n", + " root=root,\n", + " tables=[\"patients\", \"visits\"], # default tables, always loaded\n", + " dataset_name=\"my_clinic\",\n", + " config_path=config_path,\n", + " **kwargs,\n", + " )\n", + "\n", + " def preprocess_visits(self, df: nw.LazyFrame) -> nw.LazyFrame:\n", + " \"\"\"Upper-cases diagnosis codes, which the source records inconsistently.\"\"\"\n", + " return df.with_columns(nw.col(\"diagnosis\").str.to_uppercase())\n", + "\n", + " @property\n", + " def default_task(self):\n", + " return HighBloodPressureNextVisit()\n", + "\n", + "\n", + "clinic = MyClinicDataset(root=str(root), cache_dir=\"pyhealth_cache\")\n", + "print([v.diagnosis for v in clinic.get_patient(\"p1\").get_events(event_type=\"visits\")])\n", + "samples = clinic.set_task()\n", + "print(f\"{len(samples)} samples; vocabulary {samples.input_processors['diagnoses'].code_vocab}\")" + ] + }, + { + "cell_type": "markdown", + "id": "cell-07", + "metadata": {}, + "source": [ + "The `i10` from the raw file came out as `I10`, so both spellings share\n", + "one vocabulary entry.\n", + "\n", + "## Data that is not a table\n", + "\n", + "Image, signal and document collections are usually folders of files.\n", + "The trick PyHealth's own loaders use is to **build a metadata table\n", + "once** (one row per file, with its path and labels), save it under\n", + "`root`, and point the config at it. `COVID19CXRDataset` does this in\n", + "`prepare_metadata`, which reads the per-class spreadsheets, builds the\n", + "image paths and labels, checks every file exists, and writes\n", + "`covid19_cxr-metadata-pyhealth.csv`. Its config is then tiny:\n", + "\n", + "```yaml\n", + "version: \"5.0\"\n", + "tables:\n", + " covid19_cxr:\n", + " file_path: \"covid19_cxr-metadata-pyhealth.csv\"\n", + " patient_id: null # each row is its own sample\n", + " timestamp: null\n", + " attributes: [\"path\", \"url\", \"label\"]\n", + "```\n", + "\n", + "and its task returns `{\"image\": event.path, \"disease\": event.label}`\n", + "with an `image` processor that loads the file. The pattern works for\n", + "any file collection.\n", + "\n", + "## Testing with tiny synthetic data\n", + "\n", + "Every contribution needs unit tests under `tests/`. Keep them fast and\n", + "**never commit real patient data**: build a few rows in a temporary\n", + "folder, as above, and check the behaviour you care about. In the\n", + "repository this is a file such as `tests/core/test_my_clinic.py`;\n", + "here we run the same tests inline." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell-08", + "metadata": {}, + "outputs": [], + "source": [ + "import tempfile\n", + "import unittest\n", + "\n", + "\n", + "class TestMyClinicDataset(unittest.TestCase):\n", + " @classmethod\n", + " def setUpClass(cls):\n", + " cls.tmp = tempfile.TemporaryDirectory()\n", + " cls.dataset = MyClinicDataset(root=str(root), cache_dir=cls.tmp.name)\n", + "\n", + " @classmethod\n", + " def tearDownClass(cls):\n", + " cls.tmp.cleanup()\n", + "\n", + " def test_codes_are_normalised(self):\n", + " codes = [v.diagnosis for v in self.dataset.get_patient(\"p3\").get_events(event_type=\"visits\")]\n", + " self.assertEqual(codes, [\"I10\", \"N18.3\"])\n", + "\n", + " def test_default_task_labels(self):\n", + " samples = self.dataset.set_task()\n", + " labels = sorted(int(samples[i][\"high_bp_next\"]) for i in range(len(samples)))\n", + " self.assertEqual(labels, [0, 1, 1]) # p1: 148, 139 -> 1, 0; p3: 141 -> 1\n", + "\n", + " def test_patient_without_follow_up_has_no_samples(self):\n", + " self.assertEqual(HighBloodPressureNextVisit()(self.dataset.get_patient(\"p2\")), [])\n", + "\n", + "\n", + "result = unittest.main(argv=[\"tutorial\"], exit=False, verbosity=2).result\n", + "print(\"all passed:\", result.wasSuccessful())" + ] + }, + { + "cell_type": "markdown", + "id": "cell-09", + "metadata": {}, + "source": [ + "## Opening a pull request\n", + "\n", + "PyHealth's CI checks every PR against a few rules\n", + "([CONTRIBUTING.md](https://github.com/sunlabuiuc/PyHealth/blob/master/CONTRIBUTING.md)):\n", + "\n", + "- [ ] A change under `pyhealth/` comes with a change under `docs/`\n", + " (an API page or an entry in the right index) and under\n", + " `examples/` (a short script that runs your dataset or task).\n", + "- [ ] New or changed public classes and functions have a docstring with\n", + " a `>>>` usage example (see `MyClinicDataset` above).\n", + "- [ ] New code passes `ruff check` (88-character lines).\n", + "- [ ] Tests under `tests/` use synthetic data and pass:\n", + " `python -m unittest discover -t tests -s tests/core`.\n", + "\n", + "You can run the same rule check locally before pushing:\n", + "\n", + "```bash\n", + "python tools/check_pr_rules.py --base origin/master --head HEAD\n", + "```\n", + "\n", + "In the PR description, say what the dataset or task is, where the data\n", + "comes from and its license, and how you tested it. A maintainer will\n", + "review it; small, focused PRs are reviewed fastest.\n", + "\n", + "## Summary\n", + "\n", + "- New source: dataset class (config, default tables,\n", + " `preprocess_
`, `default_task`). New question: task.\n", + "- Non-tabular data: build a metadata table once, then use the same\n", + " machinery.\n", + "- Test with a few synthetic rows; never commit patient data.\n", + "- PRs need docs, an example, docstring examples, clean lint and\n", + " passing tests." + ] + }, + { + "cell_type": "markdown", + "id": "cell-10", + "metadata": {}, + "source": [ + "## Where to go next\n", + "\n", + "The whole series:\n", + "\n", + "00. Quickstart: your first clinical prediction model\n", + "01. Datasets: patients, events and your own data\n", + "02. Tasks and processors: from patients to model-ready samples\n", + "03. Training and evaluating models properly\n", + "04. Choosing a model: from logistic regression to XGBoost\n", + "05. Interpreting predictions\n", + "06. Medical codes: lookups, mappings and groupers\n", + "07. Clinical text: classification and medical coding\n", + "08. Contributing a dataset or task to PyHealth\n", + "\n", + "Questions or problems? Open an issue at https://github.com/sunlabuiuc/PyHealth/issues. If PyHealth is useful to you, a star on GitHub helps the project." + ] + } + ], + "metadata": { + "colab": { + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/tutorials/series/README.md b/examples/tutorials/series/README.md new file mode 100644 index 000000000..5ce59f8e7 --- /dev/null +++ b/examples/tutorials/series/README.md @@ -0,0 +1,33 @@ +# PyHealth tutorial series + +Nine Colab-ready notebooks, from a first model to contributing a dataset. Each +runs on a free Colab CPU runtime with public or synthetic data, so no +credentials are needed. + +| # | Notebook | Covers | +|---|---|---| +| 00 | [Quickstart](00_quickstart.ipynb) | The five steps: dataset, task, split, model, evaluation | +| 01 | [Datasets](01_datasets.ipynb) | Patients and events, YAML configs, caching, loading your own CSVs | +| 02 | [Tasks and processors](02_tasks_and_processors.ipynb) | Writing tasks, what each processor produces | +| 03 | [Training and evaluation](03_training_and_evaluation.ipynb) | Leakage-free splits, early stopping, class imbalance, calibration, bootstrap CIs | +| 04 | [Choosing a model](04_choosing_a_model.ipynb) | Logistic regression to Transformer, plus an XGBoost baseline, on one split | +| 05 | [Interpreting predictions](05_interpreting_predictions.ipynb) | TreeSHAP, Integrated Gradients, checking faithfulness | +| 06 | [Medical codes](06_medical_codes.ipynb) | Lookups, ICD-9/ICD-10 translation, CCS/ATC groupers, `code_mapping` | +| 07 | [Clinical text](07_clinical_text.ipynb) | Fine-tuning language models; specialty classification and ICD coding | +| 08 | [Contributing](08_contributing.ipynb) | Dataset classes, file-based data, tests, PR checklist | + +Open one in Colab from +`https://colab.research.google.com/github/sunlabuiuc/PyHealth/blob/master/examples/tutorials/series/.ipynb`. + +## Editing + +The notebooks are generated from `_source/tutorials/tNN_*.py`; edit those, then +rebuild (and optionally execute) with: + +```bash +python examples/tutorials/series/_source/build.py # all notebooks +python examples/tutorials/series/_source/build.py t04 --run # rebuild and execute one +``` + +`--run` needs `nbclient` and a Jupyter kernel with PyHealth installed; it skips +the install cell and stops at the first failing cell. diff --git a/examples/tutorials/series/_source/build.py b/examples/tutorials/series/_source/build.py new file mode 100644 index 000000000..8092324d6 --- /dev/null +++ b/examples/tutorials/series/_source/build.py @@ -0,0 +1,76 @@ +"""Builds the tutorial notebooks from the Python sources in tutorials/. + +Edit the tutorials/tNN_*.py files, not the .ipynb files, then rebuild: + + python examples/tutorials/series/_source/build.py # write all notebooks + python examples/tutorials/series/_source/build.py t03 # only matching ones + python examples/tutorials/series/_source/build.py t03 --run # also execute them + +``--run`` executes each notebook top to bottom with the current environment's +PyHealth (the install cell is skipped) and stops at the first error. Set +TUTORIAL_KERNEL to choose the Jupyter kernel (default "python3"). +""" + +import importlib +import os +import sys +import time +from pathlib import Path + +import nbformat + +HERE = Path(__file__).parent +OUT = HERE.parent +sys.path.insert(0, str(HERE)) +from nbkit import notebook # noqa: E402 + + +def build(name: str) -> Path: + mod = importlib.import_module(f"tutorials.{name}") + nb = notebook(mod.cells()) + path = OUT / mod.FILENAME + nbformat.write(nb, path) + return path + + +def run(path: Path, workdir: Path) -> bool: + from nbclient import NotebookClient + + nb = nbformat.read(path, as_version=4) + for cell in nb.cells: + if "install" in cell.metadata.get("tags", []): + cell.source = "# (install skipped)" + workdir.mkdir(parents=True, exist_ok=True) + client = NotebookClient( + nb, + kernel_name=os.environ.get("TUTORIAL_KERNEL", "python3"), + timeout=3600, + allow_errors=True, + resources={"metadata": {"path": str(workdir)}}, + ) + start = time.time() + with client.setup_kernel(): + for i, cell in enumerate(nb.cells): + if cell.cell_type != "code": + continue + client.execute_cell(cell, i) + errors = [o for o in cell.outputs if o.get("output_type") == "error"] + if errors: + print(f" cell {i}: {errors[0]['ename']}: {errors[0]['evalue'][:300]}") + return False + print(f" ok in {time.time() - start:.0f}s") + return True + + +if __name__ == "__main__": + wanted = [a for a in sys.argv[1:] if not a.startswith("--")] + names = sorted(p.stem for p in (HERE / "tutorials").glob("t*.py")) + if wanted: + names = [n for n in names if any(n.startswith(w) for w in wanted)] + ok = True + for name in names: + path = build(name) + print(f"wrote {path.relative_to(OUT.parent.parent)}") + if "--run" in sys.argv: + ok &= run(path, Path("tutorial_runs") / path.stem) + sys.exit(0 if ok else 1) diff --git a/examples/tutorials/series/_source/nbkit.py b/examples/tutorials/series/_source/nbkit.py new file mode 100644 index 000000000..a79542763 --- /dev/null +++ b/examples/tutorials/series/_source/nbkit.py @@ -0,0 +1,93 @@ +"""Helpers to build the PyHealth tutorial notebooks as .ipynb files. + +Each module in tutorials/ defines `FILENAME` and `cells()`; `build.py` writes +the notebooks one folder up. Cells tagged "install" are skipped when +`build.py --run` executes the notebooks against the installed PyHealth. +""" + +import textwrap + +import nbformat + +INSTALL = ( + '%pip install -q "pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git"' +) +INSTALL_NOTE = ( + "These tutorials use features from the upcoming PyHealth 2.1 release, so " + "for now we install the latest version straight from GitHub. Once 2.1 is " + 'on PyPI this becomes `pip install "pyhealth>=2.1"`.' +) + +SERIES = [ + ("00", "Quickstart: your first clinical prediction model"), + ("01", "Datasets: patients, events and your own data"), + ("02", "Tasks and processors: from patients to model-ready samples"), + ("03", "Training and evaluating models properly"), + ("04", "Choosing a model: from logistic regression to XGBoost"), + ("05", "Interpreting predictions"), + ("06", "Medical codes: lookups, mappings and groupers"), + ("07", "Clinical text: classification and medical coding"), + ("08", "Contributing a dataset or task to PyHealth"), +] + + +def md(text: str) -> nbformat.NotebookNode: + return nbformat.v4.new_markdown_cell(textwrap.dedent(text).strip("\n")) + + +def code(text: str, tags: tuple[str, ...] = ()) -> nbformat.NotebookNode: + cell = nbformat.v4.new_code_cell(textwrap.dedent(text).strip("\n")) + if tags: + cell.metadata["tags"] = list(tags) + return cell + + +def header(number: str, title: str, learn: list[str], minutes: int, prereq: str) -> list: + lines = [ + f"# PyHealth Tutorial {number}: {title}", + "", + "**What you will learn**", + *[f"- {x}" for x in learn], + "", + f"**Time:** about {minutes} minutes on a free Colab CPU runtime. ", + f"**Before this:** {prereq}", + "", + "All data used here is synthetic or public, so every cell runs without " + "credentials.", + ] + return [ + nbformat.v4.new_markdown_cell("\n".join(lines)), + md(f"## Setup\n\n{INSTALL_NOTE}"), + code(INSTALL, tags=("install",)), + ] + + +def footer(next_number: str | None) -> list: + lines = ["## Where to go next", ""] + if next_number is not None: + lines += [f"**Next:** Tutorial {next_number}, *{dict(SERIES)[next_number]}*.", ""] + lines += [ + "The whole series:", + "", + series_table(), + "", + "Questions or problems? Open an issue at " + "https://github.com/sunlabuiuc/PyHealth/issues. If PyHealth is useful to " + "you, a star on GitHub helps the project.", + ] + return [nbformat.v4.new_markdown_cell("\n".join(lines))] + + +def series_table() -> str: + return "\n".join(f"{n}. {t}" for n, t in SERIES) + + +def notebook(cells: list) -> nbformat.NotebookNode: + nb = nbformat.v4.new_notebook() + for i, cell in enumerate(cells): + cell.id = f"cell-{i:02d}" # stable ids keep rebuilds diff-free + nb.cells = cells + nb.metadata["kernelspec"] = {"name": "python3", "display_name": "Python 3", "language": "python"} + nb.metadata["language_info"] = {"name": "python"} + nb.metadata["colab"] = {"provenance": [], "toc_visible": True} + return nb diff --git a/examples/tutorials/series/_source/tutorials/__init__.py b/examples/tutorials/series/_source/tutorials/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/tutorials/series/_source/tutorials/t00_quickstart.py b/examples/tutorials/series/_source/tutorials/t00_quickstart.py new file mode 100644 index 000000000..4cc2b6081 --- /dev/null +++ b/examples/tutorials/series/_source/tutorials/t00_quickstart.py @@ -0,0 +1,214 @@ +from nbkit import code, footer, header, md + +FILENAME = "00_quickstart.ipynb" + + +def cells(): + return [ + *header( + "00", + "Quickstart: your first clinical prediction model", + [ + "The five steps every PyHealth project follows: dataset, task, split, model, evaluation", + "How to predict in-hospital mortality from diagnosis, procedure and drug codes", + "How to read the results, and why PR-AUC matters for rare outcomes", + ], + 15, + "nothing. This is the place to start.", + ), + md(""" + ## The PyHealth pipeline in one picture + + Every PyHealth project has the same five steps. Each one is a single + object, so you can swap a piece (another dataset, task or model) + without touching the rest: + + | Step | PyHealth object | What it does | + |---|---|---| + | 1. Load data | `pyhealth.datasets` (e.g. `MIMIC3Dataset`) | Reads raw tables into patients, each with a timeline of events | + | 2. Define the task | `pyhealth.tasks` (e.g. `MortalityPredictionMIMIC3`) | Turns each patient into labelled samples (features + label) | + | 3. Split | `split_by_patient` | Keeps each patient in only one of train / validation / test | + | 4. Train | `pyhealth.models` + `Trainer` | Fits a model on the training samples | + | 5. Evaluate | `Trainer.evaluate`, `pyhealth.metrics` | Scores the model on patients it has never seen | + + We will run all five on **synthetic MIMIC-III**: a public, generated + copy of the MIMIC-III intensive-care database with the same tables and + columns, but no real patients. Results on synthetic data say nothing + clinical; the point is to learn the workflow, which is identical on the + real database. + """), + md(""" + ## Step 1. Load the dataset + + `MIMIC3Dataset` reads the raw MIMIC-III tables you ask for and links + every row to its patient. `root` can be a local folder or a URL; here + it points at the public synthetic copy. + + - `tables`: which tables to load. Patients and admissions are always + loaded; we add diagnoses, procedures and prescriptions. + - `cache_dir`: where PyHealth keeps its processed copy so the next run + is fast. With real patient data, put this inside storage you control. + + The first run downloads and processes the tables, which takes a couple + of minutes. PyHealth logs each step as it goes; the long log is normal. + """), + code(""" + from pyhealth.datasets import MIMIC3Dataset + + dataset = MIMIC3Dataset( + root="https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III", + tables=["diagnoses_icd", "procedures_icd", "prescriptions"], + cache_dir="pyhealth_cache", + ) + dataset.stats() + """), + md(""" + A dataset is a collection of **patients**, and each patient is a + timeline of **events** (an admission, a diagnosis code, a drug order). + Let's look at one patient: + """), + code(""" + patient = dataset.get_patient(dataset.unique_patient_ids[0]) + print("patient:", patient.patient_id) + for event in patient.get_events()[:5]: + print(event) + """), + md(""" + Tutorial 01 explores datasets in depth. For now, the key idea: the + dataset holds *all* the data, and it does not yet know what you want to + predict. That is the task's job. + + ## Step 2. Define the task + + A **task** says what one training example looks like. PyHealth ships + ready-made tasks for common problems. `MortalityPredictionMIMIC3` + creates one sample per hospital admission: + + - **Inputs**: the diagnosis codes (`conditions`), procedure codes + (`procedures`) and drug codes (`drugs`) recorded in that admission. + - **Label** (`mortality`): 1 if the patient died in hospital during + their *next* admission, else 0. Patients with a single admission + have no "next" admission, so they produce no samples. + + `set_task` runs the task over every patient and returns a + `SampleDataset`, ready for a model. + """), + code(""" + from pyhealth.tasks import MortalityPredictionMIMIC3 + + task = MortalityPredictionMIMIC3() + print("inputs:", task.input_schema) + print("label: ", task.output_schema) + + samples = dataset.set_task(task) + print(f"{len(samples)} samples") + """), + md(""" + The schemas tell PyHealth how to turn each field into numbers: a + `"sequence"` is a list of codes, mapped to integer ids with a + vocabulary learned from the data; `"binary"` is a 0/1 label. + + Here is one sample. The codes are already integer ids, and the label is + a tensor: + """), + code(""" + sample = samples[0] + for key, value in sample.items(): + print(f"{key:12s} {value}") + """), + md(""" + Before training, check how common the outcome is. A model that always + says "survives" would be right most of the time, so accuracy alone is + misleading for rare outcomes. + """), + code(""" + labels = [int(samples[i]["mortality"]) for i in range(len(samples))] + print(f"{sum(labels)} of {len(labels)} samples are positive " + f"({sum(labels) / len(labels):.1%})") + """), + md(""" + ## Step 3. Split by patient + + One patient can contribute several samples (one per admission). If the + same patient appears in both training and test data, the model can + recognise the patient instead of learning anything general, and the + test score looks better than it really is. `split_by_patient` puts + every patient in exactly one part: 70% train, 10% validation (to pick + the best epoch), 20% test (touched only once, at the end). + """), + code(""" + from pyhealth.datasets import get_dataloader, split_by_patient + + train_ds, val_ds, test_ds = split_by_patient(samples, [0.7, 0.1, 0.2], seed=42) + print(f"train {len(train_ds)} val {len(val_ds)} test {len(test_ds)}") + + train_loader = get_dataloader(train_ds, batch_size=64, shuffle=True) + val_loader = get_dataloader(val_ds, batch_size=64, shuffle=False) + test_loader = get_dataloader(test_ds, batch_size=64, shuffle=False) + """), + md(""" + ## Step 4. Build and train a model + + PyHealth models read the schemas from the dataset, so you do not + describe the inputs again. `RNN` embeds each code, runs a recurrent + network over each field, and combines them into one prediction. + + The `Trainer` handles the training loop: batches, optimizer, evaluation + on the validation set after each epoch, and keeping the best epoch by + the metric you `monitor`. We monitor **PR-AUC** (area under the + precision-recall curve), which focuses on how well the rare positive + cases are found. + """), + code(""" + from pyhealth.models import RNN + from pyhealth.trainer import Trainer + + model = RNN(dataset=samples) + trainer = Trainer(model=model, metrics=["pr_auc", "roc_auc"]) + trainer.train( + train_dataloader=train_loader, + val_dataloader=val_loader, + epochs=10, + monitor="pr_auc", + ) + """), + md(""" + ## Step 5. Evaluate on the test patients + + The trainer has reloaded the best epoch, so we score it once on the + held-out test patients. + + How to read the numbers: + - **ROC-AUC**: the chance that a random positive case is ranked above a + random negative one. 0.5 is a coin flip, 1.0 is perfect. + - **PR-AUC**: precision averaged over recall levels. Its no-skill + baseline is the positive rate printed above, not 0.5, so compare + against that. + """), + code(""" + scores = trainer.evaluate(test_loader) + prevalence = sum(int(test_ds[i]["mortality"]) for i in range(len(test_ds))) / len(test_ds) + print(f"test ROC-AUC {scores['roc_auc']:.3f}") + print(f"test PR-AUC {scores['pr_auc']:.3f} (no-skill baseline {prevalence:.3f})") + """), + md(""" + On synthetic data the scores are close to chance: the generated + records have little real signal. On real MIMIC-III the same code learns + meaningful patterns. + + ## What you did + + ```python + dataset = MIMIC3Dataset(root=..., tables=[...]) # 1. data + samples = dataset.set_task(MortalityPredictionMIMIC3()) # 2. task + train, val, test = split_by_patient(samples, [0.7, 0.1, 0.2]) # 3. split + trainer = Trainer(model=RNN(dataset=samples)) # 4. model + trainer.train(...); trainer.evaluate(test_loader) # 5. evaluate + ``` + + To try a variation, change one line: another task + (`ReadmissionPredictionMIMIC3`), another model (`Transformer`, + `RETAIN`), or your own data (Tutorial 01). + """), + *footer("01"), + ] diff --git a/examples/tutorials/series/_source/tutorials/t01_datasets.py b/examples/tutorials/series/_source/tutorials/t01_datasets.py new file mode 100644 index 000000000..5d9f23779 --- /dev/null +++ b/examples/tutorials/series/_source/tutorials/t01_datasets.py @@ -0,0 +1,275 @@ +from nbkit import code, footer, header, md + +FILENAME = "01_datasets.ipynb" + + +def cells(): + return [ + *header( + "01", + "Datasets: patients, events and your own data", + [ + "The difference between a dataset and a task, and why PyHealth keeps them apart", + "How to explore patients and their events, filter by type, time or admission", + "What the YAML config does, and how caching works", + "How to load your own CSV files with a short config, no new class needed", + ], + 20, + "Tutorial 00 (Quickstart) is helpful but not required.", + ), + md(""" + ## A dataset is not a task + + ![Datasets feed many tasks](https://drive.google.com/uc?export=view&id=1hHJcavXqisH9JEMqEVtE4TqEg_E489l5) + + A **dataset** in PyHealth is a pool of raw patient data: who the + patients are and everything that happened to them, in time order. It + does not know what you want to predict. + + A **task** (Tutorial 02) reads that pool and builds labelled examples + for one question. The same MIMIC-III dataset serves mortality + prediction, readmission prediction, length of stay, drug + recommendation and medical coding, each a different task. + + Keeping them apart means you load and clean the data once, then ask + many questions of it. When you contribute to PyHealth, a new data + *source* belongs in `pyhealth.datasets`; a new *question* about an + existing source (say, a new MIMIC-III outcome) belongs in + `pyhealth.tasks`. + """), + md(""" + ## Loading a built-in dataset + + PyHealth has loaders for many sources, including `MIMIC3Dataset`, + `MIMIC4Dataset`, `eICUDataset`, `OMOPDataset` (any OMOP-CDM database), + and imaging, signal and text datasets. They all share the same + arguments: + + | Argument | Meaning | + |---|---| + | `root` | Folder or URL with the raw files | + | `tables` | Which tables to load (core tables such as patients and admissions are always added) | + | `cache_dir` | Where the processed copy is stored; reused on the next run | + | `dev` | `True` keeps only 1,000 patients, for quick experiments | + + We use the public synthetic MIMIC-III, with four tables. If you ran + Tutorial 00 in the same runtime, this loads from the cache in seconds. + PyHealth logs each step as it goes; the long log is normal. + """), + code(""" + from pyhealth.datasets import MIMIC3Dataset + + dataset = MIMIC3Dataset( + root="https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III", + tables=["diagnoses_icd", "procedures_icd", "prescriptions", "noteevents"], + cache_dir="pyhealth_cache", + ) + dataset.stats() + """), + md(""" + ## Patients and events + + Everything in a dataset is an **event**: one row of one table, tied to + a patient and (usually) a time. An event has: + + - `event_type`: the table it came from (`"admissions"`, `"diagnoses_icd"`, ...) + - `timestamp`: when it happened (`None` for timeless rows such as demographics) + - attributes: the table columns listed in the config, readable as + `event.icd9_code` or `event["icd9_code"]` + + A **patient** is that patient's events in time order. + """), + code(""" + patient_ids = dataset.unique_patient_ids + patient = dataset.get_patient(patient_ids[0]) + + events = patient.get_events() + print(f"patient {patient.patient_id} has {len(events)} events") + for event in events[:3]: + print(event.event_type, event.timestamp, dict(event.attr_dict)) + """), + md(""" + ### Picking out the events you need + + `get_events` takes three kinds of filters, which you can combine: + + - `event_type="diagnoses_icd"`: events from one table + - `start=` / `end=`: events within a time window + - `filters=[("hadm_id", "==", "...")]`: events whose attribute matches, + for example everything recorded during one hospital admission + """), + code(""" + # every diagnosis code this patient ever received + diagnoses = patient.get_events(event_type="diagnoses_icd") + print("diagnosis codes:", [e.icd9_code for e in diagnoses][:10]) + + # the patient's admissions, in time order + admissions = patient.get_events(event_type="admissions") + for a in admissions: + print(f"admission {a.hadm_id}: {a.timestamp} -> {a.dischtime}, died in hospital: {a.hospital_expire_flag}") + """), + code(""" + # what happened during the first admission only + first = admissions[0] + in_first = patient.get_events(event_type="prescriptions", filters=[("hadm_id", "==", first.hadm_id)]) + print(f"{len(in_first)} drug orders in admission {first.hadm_id}:", + sorted({e.drug for e in in_first})[:8]) + + # and everything from that admission's start onwards + later = patient.get_events(start=first.timestamp) + print(f"{len(later)} events from {first.timestamp} on") + """), + md(""" + For analysis, `return_df=True` gives the events as a + [polars](https://pola.rs) DataFrame instead of a list. Attribute + columns are named `
/`; the frame has a column for + every attribute of every loaded table, so select the ones you need: + """), + code(""" + df = patient.get_events(event_type="diagnoses_icd", return_df=True) + df.select(["timestamp", "diagnoses_icd/hadm_id", "diagnoses_icd/icd9_code", "diagnoses_icd/seq_num"]) + """), + md(""" + Looping over many patients is the same idea. Here we count how many + patients have more than one admission, which is what the mortality task + in Tutorial 00 needs: + """), + code(""" + from itertools import islice + + multi = 0 + checked = 0 + for pid in islice(patient_ids, 2000): + checked += 1 + if len(dataset.get_patient(pid).get_events(event_type="admissions")) > 1: + multi += 1 + print(f"{multi} of the first {checked} patients have 2+ admissions") + """), + md(""" + ## Under the hood: the YAML config + + How does PyHealth know which column is the patient id, which is the + time, and which columns to keep? Each dataset has a YAML config. Here + is part of the one used above, read straight from the installed + package: + """), + code(""" + from pathlib import Path + import pyhealth.datasets + + config = Path(pyhealth.datasets.__file__).parent / "configs" / "mimic3.yaml" + text = config.read_text() + start = text.index(" diagnoses_icd:") + print(text[:text.index(" admissions:")]) + print(text[start:text.index("\\n\\n", start)]) + """), + md(""" + Every table entry has: + + | Field | Meaning | + |---|---| + | `file_path` | The file under `root` | + | `patient_id` | The column that identifies the patient | + | `timestamp` | The column with the event time (`null` for timeless tables) | + | `attributes` | The columns to keep; they become event attributes | + | `join` (optional) | Columns to pull in from another file. `diagnoses_icd` has no time of its own, so it borrows `dischtime` from `ADMISSIONS` | + + ## Caching + + Reading and joining the raw tables is the slow part, so PyHealth + stores the result under `cache_dir` and reuses it. The cache entry is + keyed on the dataset's settings and on the config and source files: + change a table or edit the config, and you get a fresh entry instead + of stale data. Task outputs (Tutorial 02) are cached the same way. + + Two practical notes: + - Without `cache_dir`, PyHealth uses a default folder in your home + directory and prints a warning saying where. With real patient + data, always pass a `cache_dir` inside storage you control. + - To make that mandatory (for example on a shared server), set the + environment variable `PYHEALTH_REQUIRE_CACHE_DIR=1`; loading a + dataset without `cache_dir` then raises an error. + """), + md(""" + ## Your own data: CSV files plus a config + + You do not need to write a Python class to use your own data. Put each + table in a CSV file, describe them in a YAML config, and load them + with `BaseDataset`. Here we create a tiny clinic dataset with a + demographics table and a visits table. + """), + code(""" + from pathlib import Path + + root = Path("my_clinic") + root.mkdir(exist_ok=True) + + (root / "patients.csv").write_text( + "patient_id,birth_year,sex\\n" + "p1,1956,F\\n" + "p2,1971,M\\n" + "p3,1990,F\\n" + ) + (root / "visits.csv").write_text( + "patient_id,visit_time,diagnosis,systolic_bp\\n" + "p1,2023-01-05 09:30,I10,152\\n" + "p1,2023-06-12 14:00,E11.9,148\\n" + "p1,2024-02-01 10:15,I10,139\\n" + "p2,2023-03-20 08:45,J45.909,121\\n" + "p3,2023-11-02 16:20,I10,135\\n" + "p3,2024-04-18 11:05,N18.3,141\\n" + ) + (root / "my_clinic.yaml").write_text(''' + version: "1.0" + tables: + patients: + file_path: "patients.csv" + patient_id: "patient_id" + timestamp: null + attributes: ["birth_year", "sex"] + visits: + file_path: "visits.csv" + patient_id: "patient_id" + timestamp: "visit_time" + timestamp_format: "%Y-%m-%d %H:%M" + attributes: ["diagnosis", "systolic_bp"] + ''') + print(sorted(p.name for p in root.iterdir())) + """), + code(""" + from pyhealth.datasets import BaseDataset + + clinic = BaseDataset( + root=str(root), + tables=["patients", "visits"], + dataset_name="my_clinic", + config_path=str(root / "my_clinic.yaml"), + cache_dir="pyhealth_cache", + ) + clinic.stats() + + for event in clinic.get_patient("p1").get_events(event_type="visits"): + print(event.timestamp, event.diagnosis, event.systolic_bp) + """), + md(""" + That is all it takes: your data now works with every PyHealth task, + model and evaluation tool, exactly like MIMIC-III. Values are read as + text, so convert them in your task (for example + `float(event.systolic_bp)`). + + If you later want to share the loader with others, wrap it in a small + `BaseDataset` subclass and contribute it; Tutorial 08 walks through + that, using PyHealth's own MIMIC-III and COVID-19 X-ray loaders as + examples. + + ## Summary + + - A dataset holds patients; a patient is a time-ordered list of events. + - `get_events(event_type=..., start=..., end=..., filters=...)` selects + what you need; `return_df=True` gives a DataFrame. + - A YAML config maps raw files to events; `cache_dir` keeps the + processed copy. + - Your own CSVs load with `BaseDataset` and a config. + """), + *footer("02"), + ] diff --git a/examples/tutorials/series/_source/tutorials/t02_tasks.py b/examples/tutorials/series/_source/tutorials/t02_tasks.py new file mode 100644 index 000000000..71faff3a3 --- /dev/null +++ b/examples/tutorials/series/_source/tutorials/t02_tasks.py @@ -0,0 +1,298 @@ +from nbkit import code, footer, header, md + +FILENAME = "02_tasks_and_processors.ipynb" + + +def cells(): + return [ + *header( + "02", + "Tasks and processors: from patients to model-ready samples", + [ + "What a task is: a function from one patient to a list of samples", + "How input and output schemas pick a processor for each field", + "What each common processor produces (sequence, nested sequence, multi-hot, tensor, labels)", + "How to write your own task, filter patients early, and use it with any model", + ], + 25, + "Tutorial 01 (Datasets).", + ), + md(""" + ## Where tasks fit + + ![Tasks build on datasets](https://drive.google.com/uc?export=view&id=1hHJcavXqisH9JEMqEVtE4TqEg_E489l5) + + A dataset holds every patient's events. A **task** answers one + question about them: for each patient it returns zero or more + **samples**, each a dictionary of input features and a label. + + Every task is a small class with three parts: + + ```python + class MyTask(BaseTask): + task_name = "my_task" # used in cache names + input_schema = {"conditions": "sequence"} # feature -> processor + output_schema = {"label": "binary"} # label -> processor + + def __call__(self, patient) -> list[dict]: # one patient -> samples + ... + ``` + + The **schemas** name a *processor* for each field. Processors turn raw + values (lists of codes, numbers, text) into tensors, learning what they + need (such as a code vocabulary) from the data. Think of them as the + tokenizer of a language model, applied to each field. + """), + md(""" + ## Loading the data + + Same dataset as Tutorial 01, without the notes table. + """), + code(""" + from pyhealth.datasets import MIMIC3Dataset + + dataset = MIMIC3Dataset( + root="https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III", + tables=["diagnoses_icd", "procedures_icd", "prescriptions"], + cache_dir="pyhealth_cache", + ) + """), + md(""" + ## Processors, one at a time + + Before writing a task, let's see what the common processors do. We + build a toy dataset of three hand-written samples with + `create_sample_dataset`, which applies the processors exactly as + `set_task` does, then print what came out. + """), + code(""" + from pyhealth.datasets import create_sample_dataset + + toy = [ + {"patient_id": "a", "codes": ["I10", "E11"], "visits": [["I10"], ["E11", "N18"]], + "flags": ["female", "smoker"], "labs": [1.2, 140.0], "label": 1}, + {"patient_id": "b", "codes": ["J45"], "visits": [["J45"]], + "flags": ["male"], "labs": [0.9, 138.0], "label": 0}, + {"patient_id": "c", "codes": ["I10", "J45", "N18"], "visits": [["I10", "J45"], ["N18"], ["I10"]], + "flags": ["female"], "labs": [1.5, 135.0], "label": 1}, + ] + toy_ds = create_sample_dataset( + samples=toy, + input_schema={"codes": "sequence", "visits": "nested_sequence", + "flags": "multi_hot", "labs": "tensor"}, + output_schema={"label": "binary"}, + ) + for key, value in toy_ds[2].items(): + print(f"{key:8s} {value}") + """), + md(""" + Reading the output for patient `c`: + + | Field | Processor | Raw value | Becomes | + |---|---|---|---| + | `codes` | `sequence` | a list of codes | one integer id per code (ids 0 and 1 are reserved for padding and unknown codes) | + | `visits` | `nested_sequence` | a list of visits, each a list of codes | a 2-D tensor, one row per visit, padded to the longest visit | + | `flags` | `multi_hot` | a set of categories | a 0/1 vector over all categories seen | + | `labs` | `tensor` | numbers | a float tensor, unchanged | + | `label` | `binary` | 0/1 | a float tensor of shape (1,) | + + The vocabularies the processors learned are kept with the dataset, so + you can always map ids back to codes: + """), + code(""" + codes_vocab = toy_ds.input_processors["codes"].code_vocab + print("codes vocabulary:", codes_vocab) + print("flags categories:", toy_ds.input_processors["flags"].label_vocab) + + id_to_code = {i: c for c, i in codes_vocab.items()} + print("patient c's codes:", [id_to_code[int(i)] for i in toy_ds[2]["codes"]]) + """), + md(""" + Other processors you will meet: + + | Processor | Use it for | + |---|---| + | `multiclass`, `multilabel`, `regression` | labels with several classes, several labels at once, or a number | + | `timeseries` | irregular measurements resampled to a fixed grid | + | `text` | free text, for language models | + | `nested_multihot` | per-visit code sets as 0/1 rows (compact for large vocabularies) | + | `image`, `audio`, `signal` | file paths to images, audio, physiological signals | + | `raw` | values a model reads as Python objects (kept unchanged) | + + The full list is in the + [processors docs](https://pyhealth.readthedocs.io/en/latest/api/processors.html). + + ## Using a built-in task + + PyHealth ships tasks for common questions on each dataset: mortality, + readmission, length of stay, drug recommendation, medical coding and + more. Built-in tasks can take options; readmission, for example, lets + you choose the time window and whether to drop admissions of minors. + + Options matter: by default this task skips patients under 18. The + synthetic data shifts every birth date to just before the first + admission, so every patient looks like a newborn and the default would + return no samples at all. On real MIMIC-III keep the default; here we + turn it off. When a task returns nothing, check its options and the + values it reads before suspecting the data. + """), + code(""" + from datetime import timedelta + from pyhealth.tasks import ReadmissionPredictionMIMIC3 + + readmission = ReadmissionPredictionMIMIC3(window=timedelta(days=30), exclude_minors=False) + print(readmission.input_schema, "->", readmission.output_schema) + + readmit_samples = dataset.set_task(readmission) + positives = sum(int(readmit_samples[i]["readmission"]) for i in range(len(readmit_samples))) + print(f"{len(readmit_samples)} samples, {positives} readmitted within 30 days") + """), + md(""" + ## Writing your own task + + Suppose we want to predict a **long hospital stay** (more than 7 days) + at admission time, from: + + - the diagnosis codes of the current admission (`sequence`), + - the diagnosis codes of every admission so far, visit by visit + (`nested_sequence`), so a model can see the history, + - the admission type and the patient's sex (`multi_hot`), + - the number of earlier admissions (`tensor`). + + The `__call__` method is plain Python over the patient's events, using + the same `get_events` calls as Tutorial 01. + + `pre_filter` is optional. It runs once on the whole event table before + any patient is processed, so it is a cheap place to drop patients the + task can never use. Here we keep patients with at least one diagnosis. + """), + code(""" + from datetime import datetime + + import polars as pl + from pyhealth.tasks import BaseTask + + + class LongStayMIMIC3(BaseTask): + \"\"\"Predicts whether an admission lasts more than 7 days.\"\"\" + + task_name = "LongStayMIMIC3" + input_schema = { + "conditions": "sequence", + "history": "nested_sequence", + "demographics": "multi_hot", + "prior_admissions": "tensor", + } + output_schema = {"long_stay": "binary"} + + def pre_filter(self, df: pl.LazyFrame) -> pl.LazyFrame: + with_diagnoses = ( + df.filter(pl.col("event_type") == "diagnoses_icd").select("patient_id").unique() + ) + return df.join(with_diagnoses, on="patient_id", how="semi") + + def __call__(self, patient): + sex = patient.get_events(event_type="patients")[0].gender + history = [] + samples = [] + for n, adm in enumerate(patient.get_events(event_type="admissions")): + codes = [e.icd9_code for e in patient.get_events( + event_type="diagnoses_icd", filters=[("hadm_id", "==", adm.hadm_id)])] + if not codes or adm.dischtime is None: + continue + history.append(codes) + discharge = datetime.strptime(adm.dischtime, "%Y-%m-%d %H:%M:%S") + stay_days = (discharge - adm.timestamp).total_seconds() / 86400 + samples.append({ + "patient_id": patient.patient_id, + "visit_id": adm.hadm_id, + "conditions": codes, + "history": list(history), + "demographics": [f"sex={sex}", f"type={adm.admission_type}"], + "prior_admissions": [float(n)], + "long_stay": int(stay_days > 7), + }) + return samples + """), + md(""" + Two habits worth keeping in your own tasks: + + - **Only use information available at prediction time.** The inputs + above come from the current and earlier admissions; the label comes + from the discharge time, which is in the future when the prediction + is made. A feature such as "number of drugs given during the stay" + would leak the answer. + - **Test on one patient first.** A task is just a function, so call it + directly before running it on everyone: + """), + code(""" + task = LongStayMIMIC3() + for pid in dataset.unique_patient_ids[:200]: + out = task(dataset.get_patient(pid)) + if len(out) > 1: + for s in out: + print({k: s[k] for k in ("visit_id", "conditions", "demographics", "prior_admissions", "long_stay")}) + break + """), + code(""" + long_stay = dataset.set_task(task) + print(f"{len(long_stay)} samples") + sample = long_stay[0] + for key in task.input_schema | task.output_schema: + print(f"{key:17s} {tuple(sample[key].shape)} {sample[key]}") + """), + md(""" + ## The task works with any model + + Because the schemas describe the inputs, models need no extra + configuration. Here `MultimodalRNN` routes the sequence fields through + recurrent layers and the multi-hot and numeric fields through linear + layers. (Tutorial 03 covers training properly; this is a smoke test.) + """), + code(""" + from pyhealth.datasets import get_dataloader, split_by_patient + from pyhealth.models import MultimodalRNN + from pyhealth.trainer import Trainer + + train_ds, val_ds, test_ds = split_by_patient(long_stay, [0.7, 0.1, 0.2], seed=0) + trainer = Trainer(model=MultimodalRNN(dataset=long_stay), metrics=["roc_auc", "pr_auc"]) + trainer.train( + train_dataloader=get_dataloader(train_ds, batch_size=64, shuffle=True), + val_dataloader=get_dataloader(val_ds, batch_size=64), + epochs=3, + monitor="roc_auc", + ) + print(trainer.evaluate(get_dataloader(test_ds, batch_size=64))) + """), + md(""" + ## Tasks beyond patient timelines + + A "patient" is just whatever a dataset groups events by. For image or + document collections, each row can be its own patient, and the task + returns one sample per row. PyHealth's chest X-ray task, for example, + is only a few lines: + + ```python + class COVID19CXRClassification(BaseTask): + task_name = "COVID19CXRClassification" + input_schema = {"image": "image"} # a file path -> image tensor + output_schema = {"disease": "multiclass"} + + def __call__(self, patient): + event = patient.get_events(event_type="covid19_cxr")[0] + return [{"image": event.path, "disease": event.label}] + ``` + + Tutorial 07 uses the same pattern for clinical text. + + ## Summary + + - A task maps one patient to a list of sample dictionaries. + - `input_schema` / `output_schema` choose a processor per field; the + processors learn vocabularies and produce tensors. + - `pre_filter` drops unusable patients early; call the task on one + patient to debug it. + - Keep features to what is known at prediction time. + """), + *footer("03"), + ] diff --git a/examples/tutorials/series/_source/tutorials/t03_training.py b/examples/tutorials/series/_source/tutorials/t03_training.py new file mode 100644 index 000000000..6b84fa959 --- /dev/null +++ b/examples/tutorials/series/_source/tutorials/t03_training.py @@ -0,0 +1,246 @@ +from nbkit import code, footer, header, md + +FILENAME = "03_training_and_evaluation.ipynb" + + +def cells(): + return [ + *header( + "03", + "Training and evaluating models properly", + [ + "How to split so that nothing about the test patients leaks into training", + "How to control training: epochs, early stopping, learning rate, checkpoints", + "How to handle a rare outcome with a positive-class weight, and what it costs", + "How to report discrimination and calibration with patient-level confidence intervals", + ], + 25, + "Tutorials 00 and 02.", + ), + md(""" + Getting a number out of `trainer.evaluate` is easy. Getting a number + you can trust, and report, takes a few more steps. This tutorial + walks through them on the in-hospital mortality task from Tutorial 00, + a rare outcome (about 3% of samples) where the details matter most. + """), + code(""" + from pyhealth.datasets import MIMIC3Dataset + from pyhealth.tasks import MortalityPredictionMIMIC3 + + dataset = MIMIC3Dataset( + root="https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III", + tables=["diagnoses_icd", "procedures_icd", "prescriptions"], + cache_dir="pyhealth_cache", + ) + task = MortalityPredictionMIMIC3() + """), + md(""" + ## 1. Split by patient, and fit preprocessing on training patients only + + Tutorial 00 already split **by patient**, so no patient appears in two + parts. There is a second, quieter leak: the processors. When you call + `set_task(task)` and split afterwards, the code vocabularies are built + from *all* samples, including the test patients. The model then has an + embedding slot for a code that only test patients have, which a real + deployment would never have. + + Passing `split=PatientSplit(...)` to `set_task` splits first and fits + every processor on the training patients only. It returns one dataset + per part, training part first. Codes that appear only in validation or + test map to ``, exactly as unseen codes would in practice. + """), + code(""" + from pyhealth.datasets import PatientSplit + + train_ds, val_ds, test_ds = dataset.set_task( + task, split=PatientSplit(ratios=(0.7, 0.1, 0.2), seed=42) + ) + print(f"train {len(train_ds)} val {len(val_ds)} test {len(test_ds)}") + """), + md(""" + Compare with fitting on everything first. The vocabulary fitted on all + samples is larger: the extra entries are codes seen only in the + validation and test patients. + """), + code(""" + all_samples = dataset.set_task(task) + + for field in ["conditions", "procedures", "drugs"]: + v_all = len(all_samples.input_processors[field].code_vocab) + v_train = len(train_ds.input_processors[field].code_vocab) + print(f"{field:11s} vocabulary fitted on all samples {v_all:5d}, on training patients {v_train:5d}") + + unk = train_ds.input_processors["conditions"].code_vocab[""] + test_codes = [int(c) for i in range(len(test_ds)) for c in test_ds[i]["conditions"]] + print(f"{sum(c == unk for c in test_codes) / len(test_codes):.1%} of test diagnosis codes are unseen in training") + """), + md(""" + For processors that learn statistics, such as normalisation, the same + rule applies: statistics from test patients must not shape the + training inputs. With `split=` you get that for free. + + ## 2. Training with more control + + `Trainer.train` has a few options worth knowing: + + | Option | What it does | + |---|---| + | `epochs` | Maximum passes over the training data | + | `monitor`, `monitor_criterion` | Validation metric that picks the best epoch (`"max"` or `"min"`) | + | `patience` | Stop early after this many epochs without improvement | + | `optimizer_params` | e.g. `{"lr": 1e-3}` | + | `weight_decay`, `max_grad_norm` | Regularisation and gradient clipping | + + The `Trainer` itself takes `output_path` and `exp_name`; the best and + last checkpoints are saved there, and the best one is reloaded at the + end of training. + """), + code(""" + from pyhealth.datasets import get_dataloader + from pyhealth.models import RNN + from pyhealth.trainer import Trainer + + train_loader = get_dataloader(train_ds, batch_size=64, shuffle=True) + val_loader = get_dataloader(val_ds, batch_size=256) + test_loader = get_dataloader(test_ds, batch_size=256) + + METRICS = ["pr_auc", "roc_auc"] + + model = RNN(dataset=train_ds) + trainer = Trainer(model=model, metrics=METRICS, output_path="runs", exp_name="rnn_plain") + trainer.train( + train_dataloader=train_loader, + val_dataloader=val_loader, + epochs=20, + monitor="pr_auc", + patience=5, + optimizer_params={"lr": 1e-3}, + weight_decay=1e-4, + ) + """), + md(""" + ## 3. Rare outcomes: weighting the positive class + + With 3% positives, the loss is dominated by the negatives and a model + can do well by predicting low risk for everyone. A common remedy is to + weight each positive example more. `set_pos_weight("balanced", + train_ds)` sets the weight to negatives / positives in the **training** + data (never the full dataset, so test labels do not set it). + """), + code(""" + weighted = RNN(dataset=train_ds) + weighted.set_pos_weight("balanced", train_ds) + print(f"positive weight: {float(weighted.pos_weight[0]):.1f}") + + trainer_w = Trainer(model=weighted, metrics=METRICS, output_path="runs", exp_name="rnn_weighted") + trainer_w.train( + train_dataloader=train_loader, + val_dataloader=val_loader, + epochs=20, + monitor="pr_auc", + patience=5, + optimizer_params={"lr": 1e-3}, + weight_decay=1e-4, + ) + """), + md(""" + ## 4. Discrimination *and* calibration + + Ranking patients well (discrimination: ROC-AUC, PR-AUC) is half of + the story. If the predicted risks will be read as probabilities ("this + patient has a 20% risk"), they also need to be **calibrated**: + + | Metric | Ideal | Reading it | + |---|---|---| + | `brier` | 0 | mean squared error of the probabilities | + | `oe_ratio` | 1 | observed events / expected events; below 1 means risks are overestimated | + | `calibration_slope` | 1 | below 1 means predictions are too extreme | + | `calibration_intercept` | 0 | above 0 means risks are underestimated overall | + + `Trainer.inference` returns the labels and predicted probabilities; + `binary_metrics_fn` scores them. + """), + code(""" + import numpy as np + from pyhealth.metrics import binary_metrics_fn + + REPORT = ["pr_auc", "roc_auc", "brier", "oe_ratio", "calibration_slope", "calibration_intercept"] + + y_true, prob_plain, _, pids = trainer.inference(test_loader, return_patient_ids=True) + _, prob_weighted, _ = trainer_w.inference(test_loader) + y_true, prob_plain, prob_weighted = y_true.ravel(), prob_plain.ravel(), prob_weighted.ravel() + + print(f"test prevalence {y_true.mean():.3f}\\n") + print(f"{'metric':22s} {'plain':>8s} {'weighted':>9s}") + plain = binary_metrics_fn(y_true, prob_plain, metrics=REPORT) + weighted_scores = binary_metrics_fn(y_true, prob_weighted, metrics=REPORT) + for m in REPORT: + print(f"{m:22s} {plain[m]:8.3f} {weighted_scores[m]:9.3f}") + print(f"\\nmean predicted risk: plain {prob_plain.mean():.3f}, weighted {prob_weighted.mean():.3f}") + """), + md(""" + Look at the mean predicted risk and the observed / expected ratio: + weighting pushes every probability up, so the weighted model's risks + are inflated even when its ranking is similar. That is the price of + weighting. If you need calibrated risks, either train without the + weight, or recalibrate on a separate calibration set of patients. + + ## 5. Confidence intervals that respect patients + + A test set of a few hundred samples gives noisy metrics, so report an + interval. Samples from the same patient are correlated, so the + bootstrap should resample **whole patients** (`groups=patient_ids`), + not individual samples. `paired_bootstrap_diff` compares two models on + the same resampled patients, which is the right way to ask "is model A + better than model B?". + """), + code(""" + from pyhealth.metrics import bootstrap_ci, paired_bootstrap_diff + + groups = np.asarray(pids) + for name, prob in [("plain", prob_plain), ("weighted", prob_weighted)]: + for metric in ["roc_auc", "brier"]: + ci = bootstrap_ci(y_true, prob, metric, groups=groups, n_boot=500, seed=0) + print(f"{name:9s} {metric:8s} {ci['estimate']:.3f} 95% CI [{ci['lower']:.3f}, {ci['upper']:.3f}]") + + diff = paired_bootstrap_diff(y_true, prob_weighted, prob_plain, "roc_auc", groups=groups, n_boot=500, seed=0) + print(f"\\nweighted - plain ROC-AUC {diff['estimate']:+.3f} 95% CI [{diff['lower']:+.3f}, {diff['upper']:+.3f}]") + """), + md(""" + If the interval of the difference includes 0, the data cannot tell the + two models apart. With a few hundred test samples and a 3% outcome, + that is the usual result, and worth knowing before claiming a + winner. + + ## 6. Saving and reloading a model + + The trainer already saved the best checkpoint under + `runs//best.ckpt`. To reuse a model later, build the same + architecture on a dataset with the same processors, and load it: + """), + code(""" + import os + + ckpt = os.path.join(trainer.exp_path, "best.ckpt") + restored = Trainer(model=RNN(dataset=train_ds), metrics=METRICS, checkpoint_path=ckpt) + print("restored:", restored.evaluate(test_loader)) + print("original:", trainer.evaluate(test_loader)) + """), + md(""" + To use the model on new data later, also keep the fitted processors + (`pyhealth.datasets.save_processors` / `load_processors`), so new + samples get the same code ids. + + ## Summary + + - Use `set_task(task, split=PatientSplit(...))`: patients stay in one + part, and preprocessing learns from training patients only. + - Pick the best epoch on validation with `monitor` and `patience`; + touch the test set once. + - A positive-class weight can help rare outcomes but inflates + probabilities; check calibration. + - Report discrimination and calibration with patient-level bootstrap + intervals, and compare models with a paired bootstrap. + """), + *footer("04"), + ] diff --git a/examples/tutorials/series/_source/tutorials/t04_models.py b/examples/tutorials/series/_source/tutorials/t04_models.py new file mode 100644 index 000000000..0f2928f14 --- /dev/null +++ b/examples/tutorials/series/_source/tutorials/t04_models.py @@ -0,0 +1,244 @@ +from nbkit import code, footer, header, md + +FILENAME = "04_choosing_a_model.ipynb" + + +def cells(): + return [ + *header( + "04", + "Choosing a model: from logistic regression to XGBoost", + [ + "What the main PyHealth models assume about the data, and when to pick each", + "How to compare several models fairly on the same patients", + "How to add a gradient-boosted tree baseline (XGBoost) to a PyHealth benchmark", + ], + 20, + "Tutorial 03.", + ), + md(""" + ## The models at a glance + + All PyHealth models take the dataset in their constructor and read the + input and output schemas from it, so swapping models is a one-line + change. The main ones for coded EHR data: + + | Model | Idea | Good first choice when | + |---|---|---| + | `LogisticRegression` | Embeds each code, sums them, one linear layer | You want a transparent, hard-to-beat baseline | + | `MLP` | Same pooled embeddings, then hidden layers | Features interact, but order does not matter | + | `RNN` | Reads each field as a sequence (GRU or LSTM) | Order within a field carries signal | + | `Transformer` | Self-attention over each field | Longer histories, long-range interactions | + | `RETAIN` | Two attention layers over visits | You need visit- and code-level attention weights | + | `XGBoostModel` | Gradient-boosted trees on code counts or tabular features | Tabular or bag-of-codes data; the standard strong baseline | + + On tabular and bag-of-codes data, gradient-boosted trees are often as + good as neural models and much faster to train, so a benchmark without + them can overstate what a neural model adds. + + ## The task: spotting heart failure from the medication list + + We need a question the synthetic data can answer, so we use a + *phenotyping* task: does this admission include a heart-failure + diagnosis (ICD-9 428.x), judged from the drugs and procedures recorded + in it? Phenotyping models like this help find patients with a + condition when diagnosis codes are missing or unreliable, for example + when building a study cohort. About 1 in 5 admissions qualifies. + + The task is a few lines (Tutorial 02 explains each part). Drugs are + kept by name so the results stay readable. + """), + code(""" + from collections import defaultdict + + from pyhealth.tasks import BaseTask + + + class HeartFailureFromMedications(BaseTask): + \"\"\"Does an admission include a heart-failure diagnosis (ICD-9 428.x)?\"\"\" + + task_name = "HeartFailureFromMedications" + input_schema = {"drugs": "sequence", "procedures": "sequence"} + output_schema = {"heart_failure": "binary"} + + def __call__(self, patient): + # Fetch each event type once, then group by admission: much + # faster than one filtered query per admission. + by_admission = defaultdict(lambda: {"drugs": [], "diagnoses": [], "procedures": []}) + for e in patient.get_events(event_type="prescriptions"): + if e.drug: + by_admission[e.hadm_id]["drugs"].append(e.drug) + for e in patient.get_events(event_type="diagnoses_icd"): + by_admission[e.hadm_id]["diagnoses"].append(e.icd9_code) + for e in patient.get_events(event_type="procedures_icd"): + by_admission[e.hadm_id]["procedures"].append(e.icd9_code) + + samples = [] + for hadm_id, codes in by_admission.items(): + if not codes["drugs"] or not codes["diagnoses"]: + continue + samples.append({ + "patient_id": patient.patient_id, + "visit_id": hadm_id, + "drugs": codes["drugs"], + "procedures": codes["procedures"] or ["none"], + "heart_failure": int(any(c.startswith("428") for c in codes["diagnoses"])), + }) + return samples + """), + md(""" + ## One split for every model + + A fair comparison uses the same patients for every model, so we build + the samples once, split by patient (Tutorial 03), and reuse the three + parts. + """), + code(""" + from pyhealth.datasets import MIMIC3Dataset, PatientSplit, get_dataloader + + dataset = MIMIC3Dataset( + root="https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III", + tables=["diagnoses_icd", "procedures_icd", "prescriptions"], + cache_dir="pyhealth_cache", + ) + train_ds, val_ds, test_ds = dataset.set_task( + HeartFailureFromMedications(), split=PatientSplit((0.7, 0.1, 0.2), seed=7) + ) + print(f"train {len(train_ds)} val {len(val_ds)} test {len(test_ds)}") + + val_loader = get_dataloader(val_ds, batch_size=512) + test_loader = get_dataloader(test_ds, batch_size=512) + """), + md(""" + ## Neural models + + One helper trains any model the same way: same optimizer, same + maximum epochs, the best epoch picked on validation PR-AUC, and the + same test set. Only the model changes. + + Sequence models are slow on a CPU with 35,000 training admissions, so + on a CPU runtime we train the three fastest; on a GPU runtime + (Runtime > Change runtime type) all five run in a few minutes. + """), + code(""" + import time + + import numpy as np + import torch + from pyhealth.models import MLP, RETAIN, RNN, LogisticRegression, Transformer + from pyhealth.trainer import Trainer + + METRICS = ["pr_auc", "roc_auc"] + results = {} + + + def fit_and_test(name, model_class, **kwargs): + torch.manual_seed(0) + model = model_class(dataset=train_ds, **kwargs) + trainer = Trainer(model=model, metrics=METRICS, output_path="runs", exp_name=name) + start = time.time() + trainer.train( + train_dataloader=get_dataloader(train_ds, batch_size=256, shuffle=True), + val_dataloader=val_loader, + epochs=6, + monitor="pr_auc", + patience=2, + ) + scores = trainer.evaluate(test_loader) + n_params = sum(p.numel() for p in model.parameters()) + results[name] = {**scores, "seconds": time.time() - start, "parameters": n_params} + return trainer + + + models = [("LogisticRegression", LogisticRegression), ("MLP", MLP), ("RNN", RNN)] + if torch.cuda.is_available(): + models += [("Transformer", Transformer), ("RETAIN", RETAIN)] + else: + print("CPU runtime: skipping Transformer and RETAIN (about 10 minutes each on CPU). " + "Switch to a GPU runtime to include them.") + for name, cls in models: + fit_and_test(name, cls) + """), + md(""" + ## Adding XGBoost + + Gradient-boosted trees do not train with `Trainer.train`: they are fit + once on the full training set with `model.fit(...)`. After that they + work with the same `Trainer.evaluate`, metrics and checkpoints as + every other model. + + Trees need fixed-width numeric inputs. Code lists vary in length, so + `bag_of_codes=True` turns each field into per-sample code counts over + the vocabulary (one column per code). `early_stopping_rounds` uses the + validation set to choose the number of trees. + + Use unshuffled loaders for `fit`: the order of rows affects row + subsampling, so a fixed order makes the fit reproducible. + """), + code(""" + from pyhealth.models import XGBoostModel + + start = time.time() + xgb = XGBoostModel( + train_ds, + bag_of_codes=True, + n_estimators=500, + max_depth=4, + learning_rate=0.05, + subsample=0.8, + colsample_bytree=0.8, + random_state=0, + early_stopping_rounds=50, + eval_metric="aucpr", + ) + xgb.fit(get_dataloader(train_ds, batch_size=1024), val_loader) + scores = Trainer(model=xgb, metrics=METRICS, enable_logging=False).evaluate(test_loader) + results["XGBoost"] = {**scores, "seconds": time.time() - start, + "parameters": xgb.estimators_[0].best_iteration + 1} + print(f"{len(xgb.feature_names)} columns, e.g. {xgb.feature_names[:3]}") + print(f"best number of trees: {xgb.estimators_[0].best_iteration + 1}") + """), + md(""" + ## Results + """), + code(""" + prevalence = float(np.mean([int(test_ds[i]["heart_failure"]) for i in range(len(test_ds))])) + print(f"test prevalence (PR-AUC of a random guess): {prevalence:.3f}\\n") + print(f"{'model':20s} {'PR-AUC':>7s} {'ROC-AUC':>8s} {'seconds':>8s} {'size':>10s}") + for name, r in sorted(results.items(), key=lambda kv: -kv[1]["pr_auc"]): + size = f"{r['parameters']} trees" if name == "XGBoost" else f"{r['parameters']:,}" + print(f"{name:20s} {r['pr_auc']:7.3f} {r['roc_auc']:8.3f} {r['seconds']:8.1f} {size:>10s}") + """), + md(""" + ### Reading the table + + - Compare every model with the random-guess line first: PR-AUC is only + meaningful relative to the prevalence. + - Look at cost as well as score: training time and model size differ + by orders of magnitude for similar accuracy. + - Small gaps can be noise. Before + declaring a winner, compute bootstrap intervals of the paired + difference (Tutorial 03, `paired_bootstrap_diff`). + - Neural model scores also move with the random seed. Report the mean + and spread over several seeds rather than one run. + + ## Rules of thumb + + 1. Start with `LogisticRegression` and `XGBoostModel`. They are fast + and often strong, and every other model has to beat them. + 2. Move to `RNN`, `Transformer` or `RETAIN` when order or long + histories plausibly matter, and check the gain with intervals. + 3. Keep the split, the metric used to pick the epoch, and the test set + identical across models. + + ## Summary + + - Every model reads the schemas, so swapping models is one line. + - `XGBoostModel` fits with `model.fit(train, val)` and then works with + the standard `Trainer` evaluation; `bag_of_codes=True` handles code + lists. + - Compare on the same patients, report intervals, and include strong + simple baselines. + """), + *footer("05"), + ] diff --git a/examples/tutorials/series/_source/tutorials/t05_interpret.py b/examples/tutorials/series/_source/tutorials/t05_interpret.py new file mode 100644 index 000000000..1e6ba283c --- /dev/null +++ b/examples/tutorials/series/_source/tutorials/t05_interpret.py @@ -0,0 +1,286 @@ +from nbkit import code, footer, header, md + +FILENAME = "05_interpreting_predictions.ipynb" + + +def cells(): + return [ + *header( + "05", + "Interpreting predictions", + [ + "Which inputs drive a model overall (global importance) and for one patient (local explanation)", + "Exact TreeSHAP for XGBoost, and Integrated Gradients for neural models", + "How to check whether an explanation is faithful to the model, not just plausible", + ], + 20, + "Tutorial 04 (we reuse its heart-failure task).", + ), + md(""" + Clinicians reasonably ask *why* a model flags a patient. PyHealth has + two families of tools: + + - **Exact Shapley values for tree models** (`XGBoostModel.explain`, + `TreeSHAP`): fast and deterministic, computed by XGBoost itself. + - **Attribution methods for neural models** in `pyhealth.interpret`: + Integrated Gradients, DeepLIFT, GIM, attention-based methods (Chefer, + attention rollout), and model-agnostic SHAP and LIME. They share one + interface: `attribute(**batch)` returns a score per input position, + keyed by feature name. + + We use the heart-failure phenotyping task from Tutorial 04: does an + admission include a heart-failure diagnosis, judged from its drugs and + procedures? + """), + code(""" + from collections import defaultdict + + from pyhealth.datasets import MIMIC3Dataset, PatientSplit, get_dataloader + from pyhealth.tasks import BaseTask + + + class HeartFailureFromMedications(BaseTask): + \"\"\"Does an admission include a heart-failure diagnosis (ICD-9 428.x)?\"\"\" + + task_name = "HeartFailureFromMedications" + input_schema = {"drugs": "sequence", "procedures": "sequence"} + output_schema = {"heart_failure": "binary"} + + def __call__(self, patient): + # Fetch each event type once, then group by admission: much + # faster than one filtered query per admission. + by_admission = defaultdict(lambda: {"drugs": [], "diagnoses": [], "procedures": []}) + for e in patient.get_events(event_type="prescriptions"): + if e.drug: + by_admission[e.hadm_id]["drugs"].append(e.drug) + for e in patient.get_events(event_type="diagnoses_icd"): + by_admission[e.hadm_id]["diagnoses"].append(e.icd9_code) + for e in patient.get_events(event_type="procedures_icd"): + by_admission[e.hadm_id]["procedures"].append(e.icd9_code) + + samples = [] + for hadm_id, codes in by_admission.items(): + if not codes["drugs"] or not codes["diagnoses"]: + continue + samples.append({ + "patient_id": patient.patient_id, + "visit_id": hadm_id, + "drugs": codes["drugs"], + "procedures": codes["procedures"] or ["none"], + "heart_failure": int(any(c.startswith("428") for c in codes["diagnoses"])), + }) + return samples + + + dataset = MIMIC3Dataset( + root="https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III", + tables=["diagnoses_icd", "procedures_icd", "prescriptions"], + cache_dir="pyhealth_cache", + ) + train_ds, val_ds, test_ds = dataset.set_task( + HeartFailureFromMedications(), split=PatientSplit((0.7, 0.1, 0.2), seed=7) + ) + val_loader = get_dataloader(val_ds, batch_size=512) + test_loader = get_dataloader(test_ds, batch_size=512) + """), + md(""" + ## 1. Tree models: exact Shapley values + + A Shapley value says how much one input moved a prediction away from + the average prediction. For tree ensembles they can be computed + exactly. They are on the **log-odds** scale and add up: the bias plus + all contributions equals the model's raw score for that sample. + + First, fit the model (as in Tutorial 04): + """), + code(""" + from pyhealth.models import XGBoostModel + from pyhealth.trainer import Trainer + + xgb = XGBoostModel(train_ds, bag_of_codes=True, n_estimators=400, max_depth=4, + learning_rate=0.05, subsample=0.8, colsample_bytree=0.8, + random_state=0, early_stopping_rounds=50, eval_metric="aucpr") + xgb.fit(get_dataloader(train_ds, batch_size=1024), val_loader) + print(Trainer(model=xgb, metrics=["pr_auc", "roc_auc"], enable_logging=False).evaluate(test_loader)) + """), + md(""" + ### Global importance + + `mean_abs_shap` averages the size of each column's contribution over a + dataset: which inputs the model leans on most, in either direction. + Columns are named `field=code`. + """), + code(""" + ranking = xgb.mean_abs_shap(test_loader) + for name, value in list(ranking.items())[:12]: + print(f"{value:.3f} {name}") + """), + md(""" + Do these make clinical sense? Diuretics (furosemide), beta-blockers, + ACE inhibitors and digoxin are standard heart-failure drugs, so we + would expect them near the top. A ranking that does not make sense is + worth investigating: it can reveal leakage or a data problem rather + than medical insight. (The synthetic data only partly preserves real + prescribing patterns.) + + ### Explaining one admission + + `explain` gives every column's contribution for each sample. Let's take + the test admission the model is most confident about and list what + pushed its score up or down. + """), + code(""" + import torch + + batch = next(iter(test_loader)) + out = xgb.explain(**batch) + names = [n for field in xgb.feature_layout for n in out["feature_names"][field["key"]]] + contrib = torch.cat([out["attributions"][field["key"]] for field in xgb.feature_layout], dim=1) + + i = int(out["logit"][:, 0].argmax()) + prob = torch.sigmoid(out["logit"][i, 0]) + print(f"admission {batch['visit_id'][i]}: predicted heart-failure probability {prob:.2f}, " + f"true label {int(batch['heart_failure'][i])}") + order = contrib[i].argsort(descending=True) + print("\\npushes the score up:") + for j in order[:6]: + print(f" {contrib[i, j]:+.3f} {names[j]}") + print("pushes the score down:") + for j in order[-3:]: + print(f" {contrib[i, j]:+.3f} {names[j]}") + total = out["bias"][i, 0] + contrib[i].sum() if out["bias"].ndim > 1 else out["bias"][i] + contrib[i].sum() + print(f"\\nbias + contributions = {float(total):.3f}, model log-odds = {float(out['logit'][i, 0]):.3f}") + """), + md(""" + The last line checks the bookkeeping: contributions plus bias equal the + model output exactly. Note that an *absent* drug can also contribute + (its absence lowered or raised the score), which is why some names + above may not appear in this admission's list. + + `pyhealth.interpret.methods.TreeSHAP` returns the same values in the + shape of each input field, like the neural interpreters below. + + ## 2. Neural models: Integrated Gradients + + For a neural model, Integrated Gradients compares the input with a + neutral baseline (here, every code replaced by the unknown token) and + accumulates gradients along the path between them. The result is one + score per input position: per drug in the list, per procedure. + """), + code(""" + from pyhealth.models import MLP + + torch.manual_seed(0) + mlp = MLP(dataset=train_ds) + trainer = Trainer(model=mlp, metrics=["pr_auc", "roc_auc"], enable_logging=False) + trainer.train( + train_dataloader=get_dataloader(train_ds, batch_size=256, shuffle=True), + val_dataloader=val_loader, + epochs=5, + monitor="pr_auc", + ) + print(trainer.evaluate(test_loader)) + """), + code(""" + from pyhealth.interpret.methods import IntegratedGradients + + ig = IntegratedGradients(mlp, steps=32) + small = next(iter(get_dataloader(test_ds, batch_size=16))) + attributions = ig.attribute(**small) + print({k: tuple(v.shape) for k, v in attributions.items()}) + """), + md(""" + Each attribution tensor has the shape of its input: one score per drug + position. To read them, map the token ids back to drug names with the + processor's vocabulary (Tutorial 02): + """), + code(""" + vocab = train_ds.input_processors["drugs"].code_vocab + id_to_drug = {i: d for d, i in vocab.items()} + + with torch.no_grad(): + probs = mlp(**small)["y_prob"][:, 0] + k = int(probs.argmax()) + print(f"admission {small['visit_id'][k]}: probability {probs[k]:.2f}, true label {int(small['heart_failure'][k])}") + scores = attributions["drugs"][k].detach() + tokens = small["drugs"][k] + ranked = sorted(((float(s), id_to_drug[int(t)]) for s, t in zip(scores, tokens) if int(t) > 1), reverse=True) + for s, name in ranked[:8]: + print(f" {s:+.4f} {name}") + """), + md(""" + ## 3. Is the explanation faithful? + + A plausible-looking explanation is not necessarily the one the model + uses. A standard check is a **deletion test**: remove the inputs the + explanation ranks highest and see how far the prediction falls. If the + explanation is faithful, removing its top drugs should lower the + predicted risk much more than removing the same number of random + drugs. (This is the idea behind the *comprehensiveness* metric.) + + We remove a drug by replacing its id with the padding id 0, which the + model ignores. + """), + code(""" + def remove_drugs(batch, scores, k, generator=None): + \"\"\"Copy of the batch with each sample's k highest-scoring drugs set to padding (always keeps one).\"\"\" + drugs = batch["drugs"].clone() + real = drugs > 1 # skip padding (0) and unknown (1) + if generator is not None: # random ranking instead + scores = torch.rand(drugs.shape, generator=generator) + ranked = scores.masked_fill(~real, float("-inf")).argsort(dim=1, descending=True) + for row in range(drugs.shape[0]): + top = ranked[row, : min(k, int(real[row].sum()) - 1)] # keep at least one drug + drugs[row, top] = 0 + return {**batch, "drugs": drugs} + + + def risk(batch): + with torch.no_grad(): + return mlp(**batch)["y_prob"][:, 0] + + + eval_batch = next(iter(get_dataloader(test_ds, batch_size=1024))) + ig_scores = ig.attribute(**eval_batch)["drugs"].detach() + base = risk(eval_batch) + flagged = base > 0.5 # admissions the model calls positive + gen = torch.Generator().manual_seed(0) + print(f"{int(flagged.sum())} admissions predicted positive; mean risk {base[flagged].mean():.3f}\\n") + print("drugs removed drop with IG ranking drop with random ranking") + for k in [1, 3, 5]: + ig_drop = (base - risk(remove_drugs(eval_batch, ig_scores, k)))[flagged].mean() + rnd_drop = (base - risk(remove_drugs(eval_batch, ig_scores, k, generator=gen)))[flagged].mean() + print(f"{k:13d} {ig_drop:20.3f} {rnd_drop:24.3f}") + """), + md(""" + If the Integrated Gradients column falls clearly faster than the random + column, the attributions point at inputs the model really relies on. + `pyhealth.metrics.interpretability` packages this idea as + comprehensiveness and sufficiency scores for whole datasets. + + """), + md(""" + ## Choosing a method + + | Situation | Use | + |---|---| + | XGBoost or another tree model | `xgb.explain` / `mean_abs_shap` / `TreeSHAP`: exact and fast | + | Neural model on codes | `IntegratedGradients`, `DeepLift`, `GIM` | + | Transformer, want attention-based relevance | `CheferRelevance`, `AttentionRollout` | + | Any model, treat it as a black box | `ShapExplainer`, `LimeExplainer` (slower, sampling-based) | + + Whatever you use, check faithfulness as above, and treat explanations + as hypotheses to examine with clinicians, not as causal findings: they + describe the model, and the model reflects the data, biases included. + + ## Summary + + - Tree models give exact Shapley values; they add up to the model's + log-odds output. + - Neural interpreters share `attribute(**batch)` and return one score + per input position; map ids back with the processor vocabulary. + - Measure faithfulness (comprehensiveness, sufficiency) against a + random baseline before trusting an explanation. + """), + *footer("06"), + ] diff --git a/examples/tutorials/series/_source/tutorials/t06_medcode.py b/examples/tutorials/series/_source/tutorials/t06_medcode.py new file mode 100644 index 000000000..dac972f7d --- /dev/null +++ b/examples/tutorials/series/_source/tutorials/t06_medcode.py @@ -0,0 +1,176 @@ +from nbkit import code, footer, header, md + +FILENAME = "06_medical_codes.ipynb" + + +def cells(): + return [ + *header( + "06", + "Medical codes: lookups, mappings and groupers", + [ + "What the common code systems are (ICD-9, ICD-10, CCS, ATC, NDC) and how they relate", + "How to look up a code's name, parents and children", + "How to translate between ICD-9 and ICD-10 and group codes into broader categories", + "How to use a grouping inside a task to shrink the vocabulary a model must learn", + ], + 15, + "Tutorial 02 (processors and vocabularies).", + ), + md(""" + ## Why codes need care + + EHR data records diagnoses, procedures and drugs as codes from + standard systems. Some things to know: + + | System | Codes | Example | + |---|---|---| + | ICD-9-CM / ICD-10-CM | diagnoses (US; ICD-10 since 2015) | `428.0` / `I50.9` heart failure | + | ICD-9-PROC / ICD-10-PCS | procedures | `39.95` hemodialysis | + | CCS, CCSR | groups of related ICD codes (a few hundred categories) | CCS 108: congestive heart failure | + | NDC | US drug products (package level) | `00054429731` | + | ATC | drugs by anatomy and chemistry, 5 levels | `C03CA01` furosemide | + + Fine-grained codes give large, sparse vocabularies: thousands of codes, + many seen only a handful of times. Grouping them into broader + categories often helps models, and translating between versions lets + you combine data from before and after the ICD-10 switch. + + `pyhealth.medcode` has two tools: + - `InnerMap`: everything *within* one system (names, hierarchy). + - `CrossMap`: mappings *between* systems. + """), + md(""" + ## Looking codes up: `InnerMap` + + The first load of a vocabulary downloads a small table and caches it. + MIMIC stores ICD-9 codes without the dot (`4280`); lookups accept both + forms. + """), + code(""" + from pyhealth.medcode import InnerMap + + icd9 = InnerMap.load("ICD9CM") + for c in ["4280", "428.0", "25000", "486"]: + print(f"{c:6s} {icd9.lookup(c)}") + """), + md(""" + Codes form a hierarchy. Ancestors are the broader categories a code + belongs to; descendants are its more specific codes: + """), + code(""" + print("ancestors of 428.0:") + for a in icd9.get_ancestors("428.0"): + print(f" {a:8s} {icd9.lookup(a)}") + print("\\nmore specific codes under 428.2 (systolic heart failure):") + for d in icd9.get_descendants("428.2"): + print(f" {d:8s} {icd9.lookup(d)}") + """), + code(""" + atc = InnerMap.load("ATC") + for c in ["C03CA01", "C03CA", "C03", "C"]: + print(f"{c:8s} {atc.lookup(c)}") + """), + md(""" + ## Translating and grouping: `CrossMap` + + `CrossMap.load(source, target)` maps codes from one system to another. + `map` returns a list, because one code can map to several. + """), + code(""" + from pyhealth.medcode import CrossMap + + icd9_to_icd10 = CrossMap.load("ICD9CM", "ICD10CM") + icd10_to_icd9 = CrossMap.load("ICD10CM", "ICD9CM") + for c in ["428.0", "250.00", "486", "038.9"]: + print(f"ICD-9 {c:7s} -> ICD-10 {icd9_to_icd10.map(c)}") + print() + for c in ["I50.9", "E11.9", "A41.9"]: + print(f"ICD-10 {c:6s} -> ICD-9 {icd10_to_icd9.map(c)}") + """), + md(""" + Notice `038.9` (septicemia) maps to `A41.9`, which maps back to + `995.91`, not `038.9`. Translation is not a round trip, so translate + once, in one direction, and say which in your methods. + + Groupers collapse many codes into clinically meaningful categories: + """), + code(""" + to_ccs = CrossMap.load("ICD9CM", "CCSCM") + ccs = InnerMap.load("CCSCM") + for c in ["428.0", "428.22", "402.91", "250.00", "250.60"]: + group = to_ccs.map(c) + print(f"{c:7s} {icd9.lookup(c)[:45]:45s} -> CCS {group} {ccs.lookup(group[0]) if group else ''}") + + print() + to_chapter = CrossMap.load("ICD10CM", "ICD10CHAPTER") + to_ccsr = CrossMap.load("ICD10CM", "CCSR") + for c in ["I50.9", "E11.9", "J18.9"]: + print(f"{c:6s} chapter {to_chapter.map(c)} CCSR {to_ccsr.map(c)}") + """), + md(""" + Drugs work the same way: NDC product codes map to ATC classes, and + `target_kwargs={"level": n}` picks how coarse (level 3 is the + therapeutic subgroup, level 5 the single substance): + """), + code(""" + ndc_to_atc = CrossMap.load("NDC", "ATC") + ndc = "00054429731" + for level in [3, 4, 5]: + codes = ndc_to_atc.map(ndc, target_kwargs={"level": level}) + print(f"level {level}: {[(c, atc.lookup(c)) for c in codes]}") + """), + md(""" + ## Using a grouping in a task + + Every task accepts `code_mapping={field: (source, target)}`. The + processors then map each raw code before building the vocabulary, so + the model sees CCS categories and ATC classes instead of raw codes. + Let's compare the vocabulary sizes on synthetic MIMIC-III: + """), + code(""" + from pyhealth.datasets import MIMIC3Dataset + from pyhealth.tasks import MortalityPredictionMIMIC3 + + dataset = MIMIC3Dataset( + root="https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III", + tables=["diagnoses_icd", "procedures_icd", "prescriptions"], + cache_dir="pyhealth_cache", + ) + raw = dataset.set_task(MortalityPredictionMIMIC3()) + grouped = dataset.set_task(MortalityPredictionMIMIC3(code_mapping={ + "conditions": ("ICD9CM", "CCSCM"), + "procedures": ("ICD9PROC", "CCSPROC"), + "drugs": ("NDC", "ATC"), + })) + for field in ["conditions", "procedures", "drugs"]: + n_raw = len(raw.input_processors[field].code_vocab) + n_grouped = len(grouped.input_processors[field].code_vocab) + print(f"{field:11s} {n_raw:5d} raw codes -> {n_grouped:4d} groups") + """), + md(""" + A model on the grouped version has far fewer embeddings to learn, + each seen many more times. Whether that helps depends on the question: + groupings lose detail (all heart-failure subtypes become one category), + so try both and compare on validation data (Tutorial 03). + + Codes a mapping does not cover fall back to the raw code, so nothing + is silently dropped. A mapping object records what it could not map: + """), + code(""" + translator = CrossMap.load("ICD9CM", "ICD10CM") + for c in ["428.0", "999.99"]: + print(c, "->", translator.map(c)) + print("unmapped so far:", translator.unmapped_codes) + """), + md(""" + ## Summary + + - `InnerMap.load(system)`: `lookup`, `get_ancestors`, `get_descendants`. + - `CrossMap.load(source, target).map(code)`: translations and + groupers; translation is not a round trip. + - `code_mapping=` on any task groups codes before the vocabulary is + built, shrinking what the model must learn. + """), + *footer("07"), + ] diff --git a/examples/tutorials/series/_source/tutorials/t07_text.py b/examples/tutorials/series/_source/tutorials/t07_text.py new file mode 100644 index 000000000..74d317bb3 --- /dev/null +++ b/examples/tutorials/series/_source/tutorials/t07_text.py @@ -0,0 +1,204 @@ +from nbkit import code, footer, header, md + +FILENAME = "07_clinical_text.ipynb" + + +def cells(): + return [ + *header( + "07", + "Clinical text: classification and medical coding", + [ + "How to fine-tune a Hugging Face language model on clinical notes with PyHealth", + "Multiclass text classification: which specialty wrote this report?", + "Multilabel medical coding: which ICD codes does this note support?", + "Why a simple TF-IDF baseline belongs next to every language model", + ], + 25, + "Tutorials 00 and 02. A GPU runtime (Runtime > Change runtime type > T4 GPU) makes training much faster.", + ), + md(""" + ## Text is just another field + + For PyHealth, a note is one more input field. The `text` processor + keeps it as a string, and `TransformersModel` wraps any Hugging Face + encoder: it tokenizes the text (first 256 tokens), encodes it, and + adds a prediction head that matches the task's label (multiclass, + multilabel, binary). + + Model size matters on a free runtime. We pick automatically: a + clinical BERT (110M parameters, trained on MIMIC notes) on a GPU, or a + small general BERT (11M parameters) on a CPU. + """), + code(""" + import torch + + if torch.cuda.is_available(): + MODEL_NAME, EPOCHS = "emilyalsentzer/Bio_ClinicalBERT", 3 + else: + MODEL_NAME, EPOCHS = "google/bert_uncased_L-4_H-256_A-4", 2 + print("device:", "cuda" if torch.cuda.is_available() else "cpu", "| model:", MODEL_NAME) + """), + md(""" + ## Part A. Which specialty wrote this report? + + The [MTSamples](https://www.kaggle.com/datasets/tboyle10/medicaltranscriptions) + collection has about 5,000 transcribed medical reports, each labelled + with a medical specialty. These are sample documents, not patient + records, so each report is its own "patient". + """), + code(""" + import urllib.request + import zipfile + + urllib.request.urlretrieve( + "https://storage.googleapis.com/pyhealth/medical_transcriptions_data/MedicalTranscriptions.zip", + "MedicalTranscriptions.zip", + ) + zipfile.ZipFile("MedicalTranscriptions.zip").extractall(".") + + from pyhealth.datasets import MedicalTranscriptionsDataset + + mts = MedicalTranscriptionsDataset(root="MedicalTranscriptions", cache_dir="pyhealth_cache") + mts.stats() + """), + md(""" + The dataset comes with a default task, text in and specialty out: + """), + code(""" + print(mts.default_task.input_schema, "->", mts.default_task.output_schema) + reports = mts.set_task() + + sample = reports[0] + print(f"{len(reports)} reports") + print("text:", sample["transcription"][:300], "...") + labels = reports.output_processors["medical_specialty"].label_vocab + print(f"\\n{len(labels)} specialties, e.g. {list(labels)[:6]}") + """), + md(""" + Each report is its own patient, so a sample-level split is fine here. + Then the usual PyHealth steps: model, trainer, evaluation. For + multiclass labels the trainer reports accuracy and F1; macro F1 + weights every specialty equally, so rare specialties count as much as + common ones. + + Loading the model prints a table of "unexpected" weights. That is + normal: the pretraining heads of the checkpoint are dropped, and + PyHealth adds a new classification head for our labels. + """), + code(""" + from pyhealth.datasets import get_dataloader, split_by_sample + from pyhealth.models import TransformersModel + from pyhealth.trainer import Trainer + + train_ds, val_ds, test_ds = split_by_sample(reports, [0.7, 0.1, 0.2], seed=0) + model = TransformersModel(dataset=reports, model_name=MODEL_NAME) + trainer = Trainer(model=model, metrics=["accuracy", "f1_macro", "f1_weighted"]) + trainer.train( + train_dataloader=get_dataloader(train_ds, batch_size=16, shuffle=True), + val_dataloader=get_dataloader(val_ds, batch_size=32), + epochs=EPOCHS, + monitor="f1_weighted", + optimizer_params={"lr": 5e-5}, + ) + bert_scores = trainer.evaluate(get_dataloader(test_ds, batch_size=32)) + print(bert_scores) + """), + md(""" + ### A baseline to beat + + Bag-of-words models are fast and often strong on text classification. + A TF-IDF + logistic regression baseline on the same split takes + seconds; a language model should clearly beat it to justify its cost. + """), + code(""" + from sklearn.feature_extraction.text import TfidfVectorizer + from sklearn.linear_model import LogisticRegression + from sklearn.metrics import accuracy_score, f1_score + + def texts_and_labels(ds): + return [ds[i]["transcription"] for i in range(len(ds))], [int(ds[i]["medical_specialty"]) for i in range(len(ds))] + + x_train, y_train = texts_and_labels(train_ds) + x_test, y_test = texts_and_labels(test_ds) + tfidf = TfidfVectorizer(min_df=2, ngram_range=(1, 2), sublinear_tf=True) + clf = LogisticRegression(max_iter=2000).fit(tfidf.fit_transform(x_train), y_train) + pred = clf.predict(tfidf.transform(x_test)) + print(f"TF-IDF + LR accuracy {accuracy_score(y_test, pred):.3f} " + f"F1 macro {f1_score(y_test, pred, average='macro'):.3f} " + f"F1 weighted {f1_score(y_test, pred, average='weighted'):.3f}") + print(f"{MODEL_NAME.split('/')[-1]:13s} accuracy {bert_scores['accuracy']:.3f} " + f"F1 macro {bert_scores['f1_macro']:.3f} F1 weighted {bert_scores['f1_weighted']:.3f}") + """), + md(""" + Scores on this dataset are modest for any model: several "specialties" + (Surgery, Consult, Discharge Summary) describe the *kind* of document + rather than the field, and overlap heavily with the others. Looking at + the labels before modelling saves a lot of tuning. + + ## Part B. Medical coding from notes + + Medical coding assigns ICD codes to a hospital stay from its + documentation. It is a **multilabel** problem: each note supports + several codes at once. PyHealth's `MIMIC3ICD9Coding` task joins a + patient's notes into one text and collects the diagnosis and + procedure codes as labels: + """), + code(""" + from pyhealth.datasets import MIMIC3Dataset + from pyhealth.tasks import MIMIC3ICD9Coding + + mimic = MIMIC3Dataset( + root="https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III", + tables=["diagnoses_icd", "procedures_icd", "noteevents"], + cache_dir="pyhealth_cache", + ) + coding_task = MIMIC3ICD9Coding() + print(coding_task.input_schema, "->", coding_task.output_schema) + notes = mimic.set_task(coding_task) + codes = notes.output_processors["icd_codes"].label_vocab + print(f"{len(notes)} samples, {len(codes)} distinct codes") + print("first sample has", int(notes[0]["icd_codes"].sum()), "codes") + """), + md(""" + With thousands of possible codes, most appearing only a few times, + coding is hard. Two habits from practice: + + - Report **micro F1** (dominated by frequent codes) and **macro F1** + (every code equal), plus ranking metrics such as precision@k. + - Restrict to the most frequent codes (for example the top 50) for a + first model, as most published MIMIC coding benchmarks do. + + The training code is identical to Part A; only the task changes. (On + synthetic notes the text carries little information about the codes, + so expect low scores; the workflow is what transfers to real MIMIC.) + """), + code(""" + from pyhealth.datasets import split_by_patient + + train_n, val_n, test_n = split_by_patient(notes, [0.7, 0.1, 0.2], seed=0) + coder = TransformersModel(dataset=notes, model_name=MODEL_NAME) + coding_trainer = Trainer(model=coder, metrics=["f1_micro", "f1_macro", "pr_auc_samples"]) + coding_trainer.train( + train_dataloader=get_dataloader(train_n, batch_size=16, shuffle=True), + val_dataloader=get_dataloader(val_n, batch_size=32), + epochs=1, + monitor="pr_auc_samples", + optimizer_params={"lr": 5e-5}, + ) + print(coding_trainer.evaluate(get_dataloader(test_n, batch_size=32))) + """), + md(""" + ## Summary + + - A `text` field plus `TransformersModel(model_name=...)` fine-tunes + any Hugging Face encoder; the task decides multiclass vs multilabel. + - Pick the model size for your hardware; use a GPU runtime for + full-size clinical models. + - Always compare with a TF-IDF baseline, and inspect the labels before + tuning. + - Medical coding is multilabel and long-tailed: report micro and macro + metrics, and start with frequent codes. + """), + *footer("08"), + ] diff --git a/examples/tutorials/series/_source/tutorials/t08_contributing.py b/examples/tutorials/series/_source/tutorials/t08_contributing.py new file mode 100644 index 000000000..ddf218f09 --- /dev/null +++ b/examples/tutorials/series/_source/tutorials/t08_contributing.py @@ -0,0 +1,256 @@ +from nbkit import code, footer, header, md + +FILENAME = "08_contributing.ipynb" + + +def cells(): + return [ + *header( + "08", + "Contributing a dataset or task to PyHealth", + [ + "Whether your idea is a dataset, a task, or both", + "How a dataset class is built: config, default tables, per-table preprocessing, a default task", + "How to handle data that is not a table (files on disk)", + "How to test it with tiny synthetic data, and what a PR needs to pass review", + ], + 25, + "Tutorials 01 and 02.", + ), + md(""" + ## Dataset or task? + + | You have... | Contribute | + |---|---| + | A new data *source* (a database, a public collection) | a dataset in `pyhealth/datasets/` | + | A new *question* about a source PyHealth already loads | a task in `pyhealth/tasks/` | + | A new source that comes with its own benchmark question | both, with the task as the dataset's `default_task` | + + A common mistake is to write a new dataset class for a new MIMIC + question (for example, synthetic data generation from MIMIC-III). That + is a task: it reuses `MIMIC3Dataset` and adds only the logic of the + question. + + ## Anatomy of a dataset class + + In Tutorial 01 we loaded CSVs with `BaseDataset` and a YAML config. A + contributed dataset wraps the same thing in a class, so users only + pass `root`. Here it is for the small clinic data, with every hook a + real dataset uses: + + 1. A default config shipped with the class. + 2. Default tables that are always loaded (like `patients` and + `admissions` in MIMIC-III). + 3. `preprocess_
`: optional per-table cleaning, applied lazily + before events are built. + 4. `default_task`: what `dataset.set_task()` runs with no argument. + """), + code(""" + from pathlib import Path + + root = Path("my_clinic") + root.mkdir(exist_ok=True) + (root / "patients.csv").write_text( + "patient_id,birth_year,sex\\n" "p1,1956,F\\n" "p2,1971,M\\n" "p3,1990,F\\n" + ) + (root / "visits.csv").write_text( + "patient_id,visit_time,diagnosis,systolic_bp\\n" + "p1,2023-01-05 09:30,i10,152\\n" + "p1,2023-06-12 14:00,E11.9,148\\n" + "p1,2024-02-01 10:15,I10,139\\n" + "p2,2023-03-20 08:45,J45.909,121\\n" + "p3,2023-11-02 16:20,I10,135\\n" + "p3,2024-04-18 11:05,n18.3,141\\n" + ) + config_path = root / "my_clinic.yaml" + config_path.write_text(''' + version: "1.0" + tables: + patients: + file_path: "patients.csv" + patient_id: "patient_id" + timestamp: null + attributes: ["birth_year", "sex"] + visits: + file_path: "visits.csv" + patient_id: "patient_id" + timestamp: "visit_time" + timestamp_format: "%Y-%m-%d %H:%M" + attributes: ["diagnosis", "systolic_bp"] + ''') + """), + md(""" + Notice the raw data has inconsistent code case (`i10` vs `I10`). The + `preprocess_visits` hook fixes that for everyone who uses the dataset. + Hooks receive a lazy dataframe through + [narwhals](https://narwhals-dev.github.io/narwhals/), which has the + same expression syntax as polars; column names arrive lowercased. + """), + code(""" + import narwhals as nw + from pyhealth.datasets import BaseDataset + from pyhealth.tasks import BaseTask + + + class HighBloodPressureNextVisit(BaseTask): + \"\"\"Is systolic blood pressure >= 140 at the next visit?\"\"\" + + task_name = "HighBloodPressureNextVisit" + input_schema = {"diagnoses": "sequence", "last_bp": "tensor"} + output_schema = {"high_bp_next": "binary"} + + def __call__(self, patient): + visits = patient.get_events(event_type="visits") + return [ + { + "patient_id": patient.patient_id, + "diagnoses": [v.diagnosis for v in visits[: i + 1]], + "last_bp": [float(visits[i].systolic_bp)], + "high_bp_next": int(float(visits[i + 1].systolic_bp) >= 140), + } + for i in range(len(visits) - 1) + ] + + + class MyClinicDataset(BaseDataset): + \"\"\"Visits and demographics from My Clinic. + + Args: + root: Folder with patients.csv and visits.csv. + config_path: Optional config; defaults to the one shipped with the class. + **kwargs: Passed to BaseDataset (cache_dir, dev, num_workers, ...). + + Examples: + >>> dataset = MyClinicDataset(root="/path/to/my_clinic") # doctest: +SKIP + >>> samples = dataset.set_task() # doctest: +SKIP + \"\"\" + + def __init__(self, root, config_path=None, **kwargs): + # In the package this is Path(__file__).parent / "configs" / "my_clinic.yaml". + config_path = config_path or str(Path(root) / "my_clinic.yaml") + super().__init__( + root=root, + tables=["patients", "visits"], # default tables, always loaded + dataset_name="my_clinic", + config_path=config_path, + **kwargs, + ) + + def preprocess_visits(self, df: nw.LazyFrame) -> nw.LazyFrame: + \"\"\"Upper-cases diagnosis codes, which the source records inconsistently.\"\"\" + return df.with_columns(nw.col("diagnosis").str.to_uppercase()) + + @property + def default_task(self): + return HighBloodPressureNextVisit() + + + clinic = MyClinicDataset(root=str(root), cache_dir="pyhealth_cache") + print([v.diagnosis for v in clinic.get_patient("p1").get_events(event_type="visits")]) + samples = clinic.set_task() + print(f"{len(samples)} samples; vocabulary {samples.input_processors['diagnoses'].code_vocab}") + """), + md(""" + The `i10` from the raw file came out as `I10`, so both spellings share + one vocabulary entry. + + ## Data that is not a table + + Image, signal and document collections are usually folders of files. + The trick PyHealth's own loaders use is to **build a metadata table + once** (one row per file, with its path and labels), save it under + `root`, and point the config at it. `COVID19CXRDataset` does this in + `prepare_metadata`, which reads the per-class spreadsheets, builds the + image paths and labels, checks every file exists, and writes + `covid19_cxr-metadata-pyhealth.csv`. Its config is then tiny: + + ```yaml + version: "5.0" + tables: + covid19_cxr: + file_path: "covid19_cxr-metadata-pyhealth.csv" + patient_id: null # each row is its own sample + timestamp: null + attributes: ["path", "url", "label"] + ``` + + and its task returns `{"image": event.path, "disease": event.label}` + with an `image` processor that loads the file. The pattern works for + any file collection. + + ## Testing with tiny synthetic data + + Every contribution needs unit tests under `tests/`. Keep them fast and + **never commit real patient data**: build a few rows in a temporary + folder, as above, and check the behaviour you care about. In the + repository this is a file such as `tests/core/test_my_clinic.py`; + here we run the same tests inline. + """), + code(""" + import tempfile + import unittest + + + class TestMyClinicDataset(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + cls.dataset = MyClinicDataset(root=str(root), cache_dir=cls.tmp.name) + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def test_codes_are_normalised(self): + codes = [v.diagnosis for v in self.dataset.get_patient("p3").get_events(event_type="visits")] + self.assertEqual(codes, ["I10", "N18.3"]) + + def test_default_task_labels(self): + samples = self.dataset.set_task() + labels = sorted(int(samples[i]["high_bp_next"]) for i in range(len(samples))) + self.assertEqual(labels, [0, 1, 1]) # p1: 148, 139 -> 1, 0; p3: 141 -> 1 + + def test_patient_without_follow_up_has_no_samples(self): + self.assertEqual(HighBloodPressureNextVisit()(self.dataset.get_patient("p2")), []) + + + result = unittest.main(argv=["tutorial"], exit=False, verbosity=2).result + print("all passed:", result.wasSuccessful()) + """), + md(""" + ## Opening a pull request + + PyHealth's CI checks every PR against a few rules + ([CONTRIBUTING.md](https://github.com/sunlabuiuc/PyHealth/blob/master/CONTRIBUTING.md)): + + - [ ] A change under `pyhealth/` comes with a change under `docs/` + (an API page or an entry in the right index) and under + `examples/` (a short script that runs your dataset or task). + - [ ] New or changed public classes and functions have a docstring with + a `>>>` usage example (see `MyClinicDataset` above). + - [ ] New code passes `ruff check` (88-character lines). + - [ ] Tests under `tests/` use synthetic data and pass: + `python -m unittest discover -t tests -s tests/core`. + + You can run the same rule check locally before pushing: + + ```bash + python tools/check_pr_rules.py --base origin/master --head HEAD + ``` + + In the PR description, say what the dataset or task is, where the data + comes from and its license, and how you tested it. A maintainer will + review it; small, focused PRs are reviewed fastest. + + ## Summary + + - New source: dataset class (config, default tables, + `preprocess_
`, `default_task`). New question: task. + - Non-tabular data: build a metadata table once, then use the same + machinery. + - Test with a few synthetic rows; never commit patient data. + - PRs need docs, an example, docstring examples, clean lint and + passing tests. + """), + *footer(None), + ] From 8ba6ba7c14e0e34ab3938686e516c62ed3f0d359 Mon Sep 17 00:00:00 2001 From: solarsys Date: Fri, 9 Oct 2026 10:24:41 -0400 Subject: [PATCH 2/6] Tutorials: self-restarting install cell for Colab On Colab, installing PyHealth downgrades numpy (and pandas, pydantic) under the copies Colab already loaded, so the first import failed with "'numpy.ufunc' object has no attribute '__module__'", and pip printed a red dependency-resolver report. The install cell now installs quietly (showing pip's log only on failure), restarts the runtime once, and skips the install on the next Run all. Verified on Colab. Co-Authored-By: Claude Opus 5.5 --- examples/tutorials/series/00_quickstart.ipynb | 36 +++++++++++++++--- examples/tutorials/series/01_datasets.ipynb | 29 +++++++++++++- .../series/02_tasks_and_processors.ipynb | 29 +++++++++++++- .../series/03_training_and_evaluation.ipynb | 29 +++++++++++++- .../series/04_choosing_a_model.ipynb | 29 +++++++++++++- .../series/05_interpreting_predictions.ipynb | 29 +++++++++++++- .../tutorials/series/06_medical_codes.ipynb | 29 +++++++++++++- .../tutorials/series/07_clinical_text.ipynb | 29 +++++++++++++- .../tutorials/series/08_contributing.ipynb | 29 +++++++++++++- examples/tutorials/series/_source/nbkit.py | 38 ++++++++++++++++--- .../_source/tutorials/t00_quickstart.py | 7 ++-- 11 files changed, 284 insertions(+), 29 deletions(-) diff --git a/examples/tutorials/series/00_quickstart.ipynb b/examples/tutorials/series/00_quickstart.ipynb index debd35127..fe7177539 100644 --- a/examples/tutorials/series/00_quickstart.ipynb +++ b/examples/tutorials/series/00_quickstart.ipynb @@ -25,7 +25,9 @@ "source": [ "## Setup\n", "\n", - "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub. Once 2.1 is on PyPI this becomes `pip install \"pyhealth>=2.1\"`." + "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub.\n", + "\n", + "**On Colab this takes two clicks of Runtime > Run all.** The first run installs PyHealth and restarts the runtime, because Colab has already loaded older versions of some packages (such as numpy). Colab then says \"Your session crashed\": that is expected. Choose Run all again and the tutorial runs through. On your own machine, install PyHealth once (`pip install \"pyhealth[xgboost]\"`) and this cell does nothing." ] }, { @@ -39,7 +41,30 @@ }, "outputs": [], "source": [ - "%pip install -q \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"" + "# First run: installs PyHealth, then restarts the runtime once so Python\n", + "# loads the new package versions. Then choose Runtime > Run all again;\n", + "# this cell will see PyHealth is installed and skip the install.\n", + "import importlib.util\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "\n", + "PYHEALTH = \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"\n", + "\n", + "if importlib.util.find_spec(\"pyhealth\") is None:\n", + " print(\"Installing PyHealth (1-2 minutes)...\")\n", + " result = subprocess.run(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", PYHEALTH],\n", + " capture_output=True, text=True,\n", + " )\n", + " if result.returncode != 0:\n", + " print(result.stdout[-5000:], result.stderr[-5000:])\n", + " raise RuntimeError(\"Installing PyHealth failed; see the log above.\")\n", + " print(\"Installed. Restarting the runtime now; then choose Runtime > Run all again.\")\n", + " os.kill(os.getpid(), 9) # Colab restarts the runtime automatically\n", + "else:\n", + " import pyhealth\n", + " print(\"PyHealth is installed and ready.\")" ] }, { @@ -323,9 +348,10 @@ "id": "cell-20", "metadata": {}, "source": [ - "On synthetic data the scores are close to chance: the generated\n", - "records have little real signal. On real MIMIC-III the same code learns\n", - "meaningful patterns.\n", + "On synthetic data the scores are modest and change noticeably from\n", + "run to run: the generated records carry little signal, and the test\n", + "set holds only a dozen or so positive cases. On real MIMIC-III the\n", + "same code learns meaningful patterns.\n", "\n", "## What you did\n", "\n", diff --git a/examples/tutorials/series/01_datasets.ipynb b/examples/tutorials/series/01_datasets.ipynb index b9e1fce5d..43ba3121d 100644 --- a/examples/tutorials/series/01_datasets.ipynb +++ b/examples/tutorials/series/01_datasets.ipynb @@ -26,7 +26,9 @@ "source": [ "## Setup\n", "\n", - "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub. Once 2.1 is on PyPI this becomes `pip install \"pyhealth>=2.1\"`." + "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub.\n", + "\n", + "**On Colab this takes two clicks of Runtime > Run all.** The first run installs PyHealth and restarts the runtime, because Colab has already loaded older versions of some packages (such as numpy). Colab then says \"Your session crashed\": that is expected. Choose Run all again and the tutorial runs through. On your own machine, install PyHealth once (`pip install \"pyhealth[xgboost]\"`) and this cell does nothing." ] }, { @@ -40,7 +42,30 @@ }, "outputs": [], "source": [ - "%pip install -q \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"" + "# First run: installs PyHealth, then restarts the runtime once so Python\n", + "# loads the new package versions. Then choose Runtime > Run all again;\n", + "# this cell will see PyHealth is installed and skip the install.\n", + "import importlib.util\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "\n", + "PYHEALTH = \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"\n", + "\n", + "if importlib.util.find_spec(\"pyhealth\") is None:\n", + " print(\"Installing PyHealth (1-2 minutes)...\")\n", + " result = subprocess.run(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", PYHEALTH],\n", + " capture_output=True, text=True,\n", + " )\n", + " if result.returncode != 0:\n", + " print(result.stdout[-5000:], result.stderr[-5000:])\n", + " raise RuntimeError(\"Installing PyHealth failed; see the log above.\")\n", + " print(\"Installed. Restarting the runtime now; then choose Runtime > Run all again.\")\n", + " os.kill(os.getpid(), 9) # Colab restarts the runtime automatically\n", + "else:\n", + " import pyhealth\n", + " print(\"PyHealth is installed and ready.\")" ] }, { diff --git a/examples/tutorials/series/02_tasks_and_processors.ipynb b/examples/tutorials/series/02_tasks_and_processors.ipynb index 7166112bc..5ac47fc40 100644 --- a/examples/tutorials/series/02_tasks_and_processors.ipynb +++ b/examples/tutorials/series/02_tasks_and_processors.ipynb @@ -26,7 +26,9 @@ "source": [ "## Setup\n", "\n", - "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub. Once 2.1 is on PyPI this becomes `pip install \"pyhealth>=2.1\"`." + "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub.\n", + "\n", + "**On Colab this takes two clicks of Runtime > Run all.** The first run installs PyHealth and restarts the runtime, because Colab has already loaded older versions of some packages (such as numpy). Colab then says \"Your session crashed\": that is expected. Choose Run all again and the tutorial runs through. On your own machine, install PyHealth once (`pip install \"pyhealth[xgboost]\"`) and this cell does nothing." ] }, { @@ -40,7 +42,30 @@ }, "outputs": [], "source": [ - "%pip install -q \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"" + "# First run: installs PyHealth, then restarts the runtime once so Python\n", + "# loads the new package versions. Then choose Runtime > Run all again;\n", + "# this cell will see PyHealth is installed and skip the install.\n", + "import importlib.util\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "\n", + "PYHEALTH = \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"\n", + "\n", + "if importlib.util.find_spec(\"pyhealth\") is None:\n", + " print(\"Installing PyHealth (1-2 minutes)...\")\n", + " result = subprocess.run(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", PYHEALTH],\n", + " capture_output=True, text=True,\n", + " )\n", + " if result.returncode != 0:\n", + " print(result.stdout[-5000:], result.stderr[-5000:])\n", + " raise RuntimeError(\"Installing PyHealth failed; see the log above.\")\n", + " print(\"Installed. Restarting the runtime now; then choose Runtime > Run all again.\")\n", + " os.kill(os.getpid(), 9) # Colab restarts the runtime automatically\n", + "else:\n", + " import pyhealth\n", + " print(\"PyHealth is installed and ready.\")" ] }, { diff --git a/examples/tutorials/series/03_training_and_evaluation.ipynb b/examples/tutorials/series/03_training_and_evaluation.ipynb index 64c7f753e..2bba966d2 100644 --- a/examples/tutorials/series/03_training_and_evaluation.ipynb +++ b/examples/tutorials/series/03_training_and_evaluation.ipynb @@ -26,7 +26,9 @@ "source": [ "## Setup\n", "\n", - "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub. Once 2.1 is on PyPI this becomes `pip install \"pyhealth>=2.1\"`." + "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub.\n", + "\n", + "**On Colab this takes two clicks of Runtime > Run all.** The first run installs PyHealth and restarts the runtime, because Colab has already loaded older versions of some packages (such as numpy). Colab then says \"Your session crashed\": that is expected. Choose Run all again and the tutorial runs through. On your own machine, install PyHealth once (`pip install \"pyhealth[xgboost]\"`) and this cell does nothing." ] }, { @@ -40,7 +42,30 @@ }, "outputs": [], "source": [ - "%pip install -q \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"" + "# First run: installs PyHealth, then restarts the runtime once so Python\n", + "# loads the new package versions. Then choose Runtime > Run all again;\n", + "# this cell will see PyHealth is installed and skip the install.\n", + "import importlib.util\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "\n", + "PYHEALTH = \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"\n", + "\n", + "if importlib.util.find_spec(\"pyhealth\") is None:\n", + " print(\"Installing PyHealth (1-2 minutes)...\")\n", + " result = subprocess.run(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", PYHEALTH],\n", + " capture_output=True, text=True,\n", + " )\n", + " if result.returncode != 0:\n", + " print(result.stdout[-5000:], result.stderr[-5000:])\n", + " raise RuntimeError(\"Installing PyHealth failed; see the log above.\")\n", + " print(\"Installed. Restarting the runtime now; then choose Runtime > Run all again.\")\n", + " os.kill(os.getpid(), 9) # Colab restarts the runtime automatically\n", + "else:\n", + " import pyhealth\n", + " print(\"PyHealth is installed and ready.\")" ] }, { diff --git a/examples/tutorials/series/04_choosing_a_model.ipynb b/examples/tutorials/series/04_choosing_a_model.ipynb index 265ca85fc..0d33ca60f 100644 --- a/examples/tutorials/series/04_choosing_a_model.ipynb +++ b/examples/tutorials/series/04_choosing_a_model.ipynb @@ -25,7 +25,9 @@ "source": [ "## Setup\n", "\n", - "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub. Once 2.1 is on PyPI this becomes `pip install \"pyhealth>=2.1\"`." + "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub.\n", + "\n", + "**On Colab this takes two clicks of Runtime > Run all.** The first run installs PyHealth and restarts the runtime, because Colab has already loaded older versions of some packages (such as numpy). Colab then says \"Your session crashed\": that is expected. Choose Run all again and the tutorial runs through. On your own machine, install PyHealth once (`pip install \"pyhealth[xgboost]\"`) and this cell does nothing." ] }, { @@ -39,7 +41,30 @@ }, "outputs": [], "source": [ - "%pip install -q \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"" + "# First run: installs PyHealth, then restarts the runtime once so Python\n", + "# loads the new package versions. Then choose Runtime > Run all again;\n", + "# this cell will see PyHealth is installed and skip the install.\n", + "import importlib.util\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "\n", + "PYHEALTH = \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"\n", + "\n", + "if importlib.util.find_spec(\"pyhealth\") is None:\n", + " print(\"Installing PyHealth (1-2 minutes)...\")\n", + " result = subprocess.run(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", PYHEALTH],\n", + " capture_output=True, text=True,\n", + " )\n", + " if result.returncode != 0:\n", + " print(result.stdout[-5000:], result.stderr[-5000:])\n", + " raise RuntimeError(\"Installing PyHealth failed; see the log above.\")\n", + " print(\"Installed. Restarting the runtime now; then choose Runtime > Run all again.\")\n", + " os.kill(os.getpid(), 9) # Colab restarts the runtime automatically\n", + "else:\n", + " import pyhealth\n", + " print(\"PyHealth is installed and ready.\")" ] }, { diff --git a/examples/tutorials/series/05_interpreting_predictions.ipynb b/examples/tutorials/series/05_interpreting_predictions.ipynb index fb42e0506..adbfeb7a3 100644 --- a/examples/tutorials/series/05_interpreting_predictions.ipynb +++ b/examples/tutorials/series/05_interpreting_predictions.ipynb @@ -25,7 +25,9 @@ "source": [ "## Setup\n", "\n", - "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub. Once 2.1 is on PyPI this becomes `pip install \"pyhealth>=2.1\"`." + "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub.\n", + "\n", + "**On Colab this takes two clicks of Runtime > Run all.** The first run installs PyHealth and restarts the runtime, because Colab has already loaded older versions of some packages (such as numpy). Colab then says \"Your session crashed\": that is expected. Choose Run all again and the tutorial runs through. On your own machine, install PyHealth once (`pip install \"pyhealth[xgboost]\"`) and this cell does nothing." ] }, { @@ -39,7 +41,30 @@ }, "outputs": [], "source": [ - "%pip install -q \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"" + "# First run: installs PyHealth, then restarts the runtime once so Python\n", + "# loads the new package versions. Then choose Runtime > Run all again;\n", + "# this cell will see PyHealth is installed and skip the install.\n", + "import importlib.util\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "\n", + "PYHEALTH = \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"\n", + "\n", + "if importlib.util.find_spec(\"pyhealth\") is None:\n", + " print(\"Installing PyHealth (1-2 minutes)...\")\n", + " result = subprocess.run(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", PYHEALTH],\n", + " capture_output=True, text=True,\n", + " )\n", + " if result.returncode != 0:\n", + " print(result.stdout[-5000:], result.stderr[-5000:])\n", + " raise RuntimeError(\"Installing PyHealth failed; see the log above.\")\n", + " print(\"Installed. Restarting the runtime now; then choose Runtime > Run all again.\")\n", + " os.kill(os.getpid(), 9) # Colab restarts the runtime automatically\n", + "else:\n", + " import pyhealth\n", + " print(\"PyHealth is installed and ready.\")" ] }, { diff --git a/examples/tutorials/series/06_medical_codes.ipynb b/examples/tutorials/series/06_medical_codes.ipynb index 1462ce3f1..2bdd6da72 100644 --- a/examples/tutorials/series/06_medical_codes.ipynb +++ b/examples/tutorials/series/06_medical_codes.ipynb @@ -26,7 +26,9 @@ "source": [ "## Setup\n", "\n", - "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub. Once 2.1 is on PyPI this becomes `pip install \"pyhealth>=2.1\"`." + "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub.\n", + "\n", + "**On Colab this takes two clicks of Runtime > Run all.** The first run installs PyHealth and restarts the runtime, because Colab has already loaded older versions of some packages (such as numpy). Colab then says \"Your session crashed\": that is expected. Choose Run all again and the tutorial runs through. On your own machine, install PyHealth once (`pip install \"pyhealth[xgboost]\"`) and this cell does nothing." ] }, { @@ -40,7 +42,30 @@ }, "outputs": [], "source": [ - "%pip install -q \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"" + "# First run: installs PyHealth, then restarts the runtime once so Python\n", + "# loads the new package versions. Then choose Runtime > Run all again;\n", + "# this cell will see PyHealth is installed and skip the install.\n", + "import importlib.util\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "\n", + "PYHEALTH = \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"\n", + "\n", + "if importlib.util.find_spec(\"pyhealth\") is None:\n", + " print(\"Installing PyHealth (1-2 minutes)...\")\n", + " result = subprocess.run(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", PYHEALTH],\n", + " capture_output=True, text=True,\n", + " )\n", + " if result.returncode != 0:\n", + " print(result.stdout[-5000:], result.stderr[-5000:])\n", + " raise RuntimeError(\"Installing PyHealth failed; see the log above.\")\n", + " print(\"Installed. Restarting the runtime now; then choose Runtime > Run all again.\")\n", + " os.kill(os.getpid(), 9) # Colab restarts the runtime automatically\n", + "else:\n", + " import pyhealth\n", + " print(\"PyHealth is installed and ready.\")" ] }, { diff --git a/examples/tutorials/series/07_clinical_text.ipynb b/examples/tutorials/series/07_clinical_text.ipynb index 943eff0e4..a2ee0a4f4 100644 --- a/examples/tutorials/series/07_clinical_text.ipynb +++ b/examples/tutorials/series/07_clinical_text.ipynb @@ -26,7 +26,9 @@ "source": [ "## Setup\n", "\n", - "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub. Once 2.1 is on PyPI this becomes `pip install \"pyhealth>=2.1\"`." + "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub.\n", + "\n", + "**On Colab this takes two clicks of Runtime > Run all.** The first run installs PyHealth and restarts the runtime, because Colab has already loaded older versions of some packages (such as numpy). Colab then says \"Your session crashed\": that is expected. Choose Run all again and the tutorial runs through. On your own machine, install PyHealth once (`pip install \"pyhealth[xgboost]\"`) and this cell does nothing." ] }, { @@ -40,7 +42,30 @@ }, "outputs": [], "source": [ - "%pip install -q \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"" + "# First run: installs PyHealth, then restarts the runtime once so Python\n", + "# loads the new package versions. Then choose Runtime > Run all again;\n", + "# this cell will see PyHealth is installed and skip the install.\n", + "import importlib.util\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "\n", + "PYHEALTH = \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"\n", + "\n", + "if importlib.util.find_spec(\"pyhealth\") is None:\n", + " print(\"Installing PyHealth (1-2 minutes)...\")\n", + " result = subprocess.run(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", PYHEALTH],\n", + " capture_output=True, text=True,\n", + " )\n", + " if result.returncode != 0:\n", + " print(result.stdout[-5000:], result.stderr[-5000:])\n", + " raise RuntimeError(\"Installing PyHealth failed; see the log above.\")\n", + " print(\"Installed. Restarting the runtime now; then choose Runtime > Run all again.\")\n", + " os.kill(os.getpid(), 9) # Colab restarts the runtime automatically\n", + "else:\n", + " import pyhealth\n", + " print(\"PyHealth is installed and ready.\")" ] }, { diff --git a/examples/tutorials/series/08_contributing.ipynb b/examples/tutorials/series/08_contributing.ipynb index 41e62ab9f..97f5824e4 100644 --- a/examples/tutorials/series/08_contributing.ipynb +++ b/examples/tutorials/series/08_contributing.ipynb @@ -26,7 +26,9 @@ "source": [ "## Setup\n", "\n", - "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub. Once 2.1 is on PyPI this becomes `pip install \"pyhealth>=2.1\"`." + "These tutorials use features from the upcoming PyHealth 2.1 release, so for now we install the latest version straight from GitHub.\n", + "\n", + "**On Colab this takes two clicks of Runtime > Run all.** The first run installs PyHealth and restarts the runtime, because Colab has already loaded older versions of some packages (such as numpy). Colab then says \"Your session crashed\": that is expected. Choose Run all again and the tutorial runs through. On your own machine, install PyHealth once (`pip install \"pyhealth[xgboost]\"`) and this cell does nothing." ] }, { @@ -40,7 +42,30 @@ }, "outputs": [], "source": [ - "%pip install -q \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"" + "# First run: installs PyHealth, then restarts the runtime once so Python\n", + "# loads the new package versions. Then choose Runtime > Run all again;\n", + "# this cell will see PyHealth is installed and skip the install.\n", + "import importlib.util\n", + "import os\n", + "import subprocess\n", + "import sys\n", + "\n", + "PYHEALTH = \"pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git\"\n", + "\n", + "if importlib.util.find_spec(\"pyhealth\") is None:\n", + " print(\"Installing PyHealth (1-2 minutes)...\")\n", + " result = subprocess.run(\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", PYHEALTH],\n", + " capture_output=True, text=True,\n", + " )\n", + " if result.returncode != 0:\n", + " print(result.stdout[-5000:], result.stderr[-5000:])\n", + " raise RuntimeError(\"Installing PyHealth failed; see the log above.\")\n", + " print(\"Installed. Restarting the runtime now; then choose Runtime > Run all again.\")\n", + " os.kill(os.getpid(), 9) # Colab restarts the runtime automatically\n", + "else:\n", + " import pyhealth\n", + " print(\"PyHealth is installed and ready.\")" ] }, { diff --git a/examples/tutorials/series/_source/nbkit.py b/examples/tutorials/series/_source/nbkit.py index a79542763..3ccff2d26 100644 --- a/examples/tutorials/series/_source/nbkit.py +++ b/examples/tutorials/series/_source/nbkit.py @@ -9,13 +9,41 @@ import nbformat -INSTALL = ( - '%pip install -q "pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git"' -) +PYHEALTH_SPEC = "pyhealth[xgboost] @ git+https://github.com/sunlabuiuc/PyHealth.git" + +INSTALL = f'''# First run: installs PyHealth, then restarts the runtime once so Python +# loads the new package versions. Then choose Runtime > Run all again; +# this cell will see PyHealth is installed and skip the install. +import importlib.util +import os +import subprocess +import sys + +PYHEALTH = "{PYHEALTH_SPEC}" + +if importlib.util.find_spec("pyhealth") is None: + print("Installing PyHealth (1-2 minutes)...") + result = subprocess.run( + [sys.executable, "-m", "pip", "install", "-q", PYHEALTH], + capture_output=True, text=True, + ) + if result.returncode != 0: + print(result.stdout[-5000:], result.stderr[-5000:]) + raise RuntimeError("Installing PyHealth failed; see the log above.") + print("Installed. Restarting the runtime now; then choose Runtime > Run all again.") + os.kill(os.getpid(), 9) # Colab restarts the runtime automatically +else: + import pyhealth + print("PyHealth is installed and ready.")''' INSTALL_NOTE = ( "These tutorials use features from the upcoming PyHealth 2.1 release, so " - "for now we install the latest version straight from GitHub. Once 2.1 is " - 'on PyPI this becomes `pip install "pyhealth>=2.1"`.' + "for now we install the latest version straight from GitHub.\n\n" + "**On Colab this takes two clicks of Runtime > Run all.** The first run " + "installs PyHealth and restarts the runtime, because Colab has already " + "loaded older versions of some packages (such as numpy). Colab then says " + '"Your session crashed": that is expected. Choose Run all again and the ' + "tutorial runs through. On your own machine, install PyHealth once " + '(`pip install "pyhealth[xgboost]"`) and this cell does nothing.' ) SERIES = [ diff --git a/examples/tutorials/series/_source/tutorials/t00_quickstart.py b/examples/tutorials/series/_source/tutorials/t00_quickstart.py index 4cc2b6081..2c66475e7 100644 --- a/examples/tutorials/series/_source/tutorials/t00_quickstart.py +++ b/examples/tutorials/series/_source/tutorials/t00_quickstart.py @@ -192,9 +192,10 @@ def cells(): print(f"test PR-AUC {scores['pr_auc']:.3f} (no-skill baseline {prevalence:.3f})") """), md(""" - On synthetic data the scores are close to chance: the generated - records have little real signal. On real MIMIC-III the same code learns - meaningful patterns. + On synthetic data the scores are modest and change noticeably from + run to run: the generated records carry little signal, and the test + set holds only a dozen or so positive cases. On real MIMIC-III the + same code learns meaningful patterns. ## What you did From 76f68dedf48e848c821b302b695afb526c861529 Mon Sep 17 00:00:00 2001 From: solarsys Date: Fri, 9 Oct 2026 10:38:15 -0400 Subject: [PATCH 3/6] Tutorials: show each PyHealth log line once on Colab PyHealth's logger has its own stdout handler and also propagates to the root logger, which Colab configures, so every message printed twice. Co-Authored-By: Claude Opus 5.5 --- examples/tutorials/series/00_quickstart.ipynb | 2 ++ examples/tutorials/series/01_datasets.ipynb | 2 ++ examples/tutorials/series/02_tasks_and_processors.ipynb | 2 ++ examples/tutorials/series/03_training_and_evaluation.ipynb | 2 ++ examples/tutorials/series/04_choosing_a_model.ipynb | 2 ++ examples/tutorials/series/05_interpreting_predictions.ipynb | 2 ++ examples/tutorials/series/06_medical_codes.ipynb | 2 ++ examples/tutorials/series/07_clinical_text.ipynb | 2 ++ examples/tutorials/series/08_contributing.ipynb | 2 ++ examples/tutorials/series/_source/nbkit.py | 2 ++ 10 files changed, 20 insertions(+) diff --git a/examples/tutorials/series/00_quickstart.ipynb b/examples/tutorials/series/00_quickstart.ipynb index fe7177539..5628a427e 100644 --- a/examples/tutorials/series/00_quickstart.ipynb +++ b/examples/tutorials/series/00_quickstart.ipynb @@ -63,7 +63,9 @@ " print(\"Installed. Restarting the runtime now; then choose Runtime > Run all again.\")\n", " os.kill(os.getpid(), 9) # Colab restarts the runtime automatically\n", "else:\n", + " import logging\n", " import pyhealth\n", + " logging.getLogger(\"pyhealth\").propagate = False # print each log line once\n", " print(\"PyHealth is installed and ready.\")" ] }, diff --git a/examples/tutorials/series/01_datasets.ipynb b/examples/tutorials/series/01_datasets.ipynb index 43ba3121d..86fc530da 100644 --- a/examples/tutorials/series/01_datasets.ipynb +++ b/examples/tutorials/series/01_datasets.ipynb @@ -64,7 +64,9 @@ " print(\"Installed. Restarting the runtime now; then choose Runtime > Run all again.\")\n", " os.kill(os.getpid(), 9) # Colab restarts the runtime automatically\n", "else:\n", + " import logging\n", " import pyhealth\n", + " logging.getLogger(\"pyhealth\").propagate = False # print each log line once\n", " print(\"PyHealth is installed and ready.\")" ] }, diff --git a/examples/tutorials/series/02_tasks_and_processors.ipynb b/examples/tutorials/series/02_tasks_and_processors.ipynb index 5ac47fc40..9693c27c0 100644 --- a/examples/tutorials/series/02_tasks_and_processors.ipynb +++ b/examples/tutorials/series/02_tasks_and_processors.ipynb @@ -64,7 +64,9 @@ " print(\"Installed. Restarting the runtime now; then choose Runtime > Run all again.\")\n", " os.kill(os.getpid(), 9) # Colab restarts the runtime automatically\n", "else:\n", + " import logging\n", " import pyhealth\n", + " logging.getLogger(\"pyhealth\").propagate = False # print each log line once\n", " print(\"PyHealth is installed and ready.\")" ] }, diff --git a/examples/tutorials/series/03_training_and_evaluation.ipynb b/examples/tutorials/series/03_training_and_evaluation.ipynb index 2bba966d2..aad3c97e6 100644 --- a/examples/tutorials/series/03_training_and_evaluation.ipynb +++ b/examples/tutorials/series/03_training_and_evaluation.ipynb @@ -64,7 +64,9 @@ " print(\"Installed. Restarting the runtime now; then choose Runtime > Run all again.\")\n", " os.kill(os.getpid(), 9) # Colab restarts the runtime automatically\n", "else:\n", + " import logging\n", " import pyhealth\n", + " logging.getLogger(\"pyhealth\").propagate = False # print each log line once\n", " print(\"PyHealth is installed and ready.\")" ] }, diff --git a/examples/tutorials/series/04_choosing_a_model.ipynb b/examples/tutorials/series/04_choosing_a_model.ipynb index 0d33ca60f..70faaadc1 100644 --- a/examples/tutorials/series/04_choosing_a_model.ipynb +++ b/examples/tutorials/series/04_choosing_a_model.ipynb @@ -63,7 +63,9 @@ " print(\"Installed. Restarting the runtime now; then choose Runtime > Run all again.\")\n", " os.kill(os.getpid(), 9) # Colab restarts the runtime automatically\n", "else:\n", + " import logging\n", " import pyhealth\n", + " logging.getLogger(\"pyhealth\").propagate = False # print each log line once\n", " print(\"PyHealth is installed and ready.\")" ] }, diff --git a/examples/tutorials/series/05_interpreting_predictions.ipynb b/examples/tutorials/series/05_interpreting_predictions.ipynb index adbfeb7a3..6f59082d9 100644 --- a/examples/tutorials/series/05_interpreting_predictions.ipynb +++ b/examples/tutorials/series/05_interpreting_predictions.ipynb @@ -63,7 +63,9 @@ " print(\"Installed. Restarting the runtime now; then choose Runtime > Run all again.\")\n", " os.kill(os.getpid(), 9) # Colab restarts the runtime automatically\n", "else:\n", + " import logging\n", " import pyhealth\n", + " logging.getLogger(\"pyhealth\").propagate = False # print each log line once\n", " print(\"PyHealth is installed and ready.\")" ] }, diff --git a/examples/tutorials/series/06_medical_codes.ipynb b/examples/tutorials/series/06_medical_codes.ipynb index 2bdd6da72..5cad0c762 100644 --- a/examples/tutorials/series/06_medical_codes.ipynb +++ b/examples/tutorials/series/06_medical_codes.ipynb @@ -64,7 +64,9 @@ " print(\"Installed. Restarting the runtime now; then choose Runtime > Run all again.\")\n", " os.kill(os.getpid(), 9) # Colab restarts the runtime automatically\n", "else:\n", + " import logging\n", " import pyhealth\n", + " logging.getLogger(\"pyhealth\").propagate = False # print each log line once\n", " print(\"PyHealth is installed and ready.\")" ] }, diff --git a/examples/tutorials/series/07_clinical_text.ipynb b/examples/tutorials/series/07_clinical_text.ipynb index a2ee0a4f4..c8bc963aa 100644 --- a/examples/tutorials/series/07_clinical_text.ipynb +++ b/examples/tutorials/series/07_clinical_text.ipynb @@ -64,7 +64,9 @@ " print(\"Installed. Restarting the runtime now; then choose Runtime > Run all again.\")\n", " os.kill(os.getpid(), 9) # Colab restarts the runtime automatically\n", "else:\n", + " import logging\n", " import pyhealth\n", + " logging.getLogger(\"pyhealth\").propagate = False # print each log line once\n", " print(\"PyHealth is installed and ready.\")" ] }, diff --git a/examples/tutorials/series/08_contributing.ipynb b/examples/tutorials/series/08_contributing.ipynb index 97f5824e4..371ab0e46 100644 --- a/examples/tutorials/series/08_contributing.ipynb +++ b/examples/tutorials/series/08_contributing.ipynb @@ -64,7 +64,9 @@ " print(\"Installed. Restarting the runtime now; then choose Runtime > Run all again.\")\n", " os.kill(os.getpid(), 9) # Colab restarts the runtime automatically\n", "else:\n", + " import logging\n", " import pyhealth\n", + " logging.getLogger(\"pyhealth\").propagate = False # print each log line once\n", " print(\"PyHealth is installed and ready.\")" ] }, diff --git a/examples/tutorials/series/_source/nbkit.py b/examples/tutorials/series/_source/nbkit.py index 3ccff2d26..67904e805 100644 --- a/examples/tutorials/series/_source/nbkit.py +++ b/examples/tutorials/series/_source/nbkit.py @@ -33,7 +33,9 @@ print("Installed. Restarting the runtime now; then choose Runtime > Run all again.") os.kill(os.getpid(), 9) # Colab restarts the runtime automatically else: + import logging import pyhealth + logging.getLogger("pyhealth").propagate = False # print each log line once print("PyHealth is installed and ready.")''' INSTALL_NOTE = ( "These tutorials use features from the upcoming PyHealth 2.1 release, so " From 4369880ca727a7f00f2a4fb671e17894db396f74 Mon Sep 17 00:00:00 2001 From: solarsys Date: Fri, 9 Oct 2026 11:18:29 -0400 Subject: [PATCH 4/6] Tutorial 04: train only the pooled models on a CPU runtime On Colab's free 2-CPU runtime the RNN on the heart-failure task needs ~7 minutes per epoch (long medication lists); sequence models now run only on a GPU runtime. Co-Authored-By: Claude Opus 5.5 --- .../series/04_choosing_a_model.ipynb | 21 +++++++++++-------- .../series/_source/tutorials/t04_models.py | 21 +++++++++++-------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/examples/tutorials/series/04_choosing_a_model.ipynb b/examples/tutorials/series/04_choosing_a_model.ipynb index 70faaadc1..01b1fc908 100644 --- a/examples/tutorials/series/04_choosing_a_model.ipynb +++ b/examples/tutorials/series/04_choosing_a_model.ipynb @@ -12,8 +12,8 @@ "- How to compare several models fairly on the same patients\n", "- How to add a gradient-boosted tree baseline (XGBoost) to a PyHealth benchmark\n", "\n", - "**Time:** about 20 minutes on a free Colab CPU runtime. \n", - "**Before this:** Tutorial 03.\n", + "**Time:** about 15 minutes on a free Colab CPU runtime. \n", + "**Before this:** Tutorial 03. A GPU runtime adds the sequence models.\n", "\n", "All data used here is synthetic or public, so every cell runs without credentials." ] @@ -197,9 +197,11 @@ "maximum epochs, the best epoch picked on validation PR-AUC, and the\n", "same test set. Only the model changes.\n", "\n", - "Sequence models are slow on a CPU with 35,000 training admissions, so\n", - "on a CPU runtime we train the three fastest; on a GPU runtime\n", - "(Runtime > Change runtime type) all five run in a few minutes." + "Sequence models read every drug of every admission in order, which is\n", + "slow on a free CPU runtime (more than 30 minutes each for 35,000\n", + "admissions). On a CPU runtime we therefore train the two pooled\n", + "models; on a GPU runtime (Runtime > Change runtime type > T4 GPU)\n", + "all five neural models run in a few minutes." ] }, { @@ -238,12 +240,13 @@ " return trainer\n", "\n", "\n", - "models = [(\"LogisticRegression\", LogisticRegression), (\"MLP\", MLP), (\"RNN\", RNN)]\n", + "models = [(\"LogisticRegression\", LogisticRegression), (\"MLP\", MLP)]\n", "if torch.cuda.is_available():\n", - " models += [(\"Transformer\", Transformer), (\"RETAIN\", RETAIN)]\n", + " models += [(\"RNN\", RNN), (\"Transformer\", Transformer), (\"RETAIN\", RETAIN)]\n", "else:\n", - " print(\"CPU runtime: skipping Transformer and RETAIN (about 10 minutes each on CPU). \"\n", - " \"Switch to a GPU runtime to include them.\")\n", + " print(\"CPU runtime: training LogisticRegression and MLP only. The sequence models \"\n", + " \"(RNN, Transformer, RETAIN) take 30+ minutes each on a free CPU runtime; \"\n", + " \"switch to a GPU runtime to include them.\")\n", "for name, cls in models:\n", " fit_and_test(name, cls)" ] diff --git a/examples/tutorials/series/_source/tutorials/t04_models.py b/examples/tutorials/series/_source/tutorials/t04_models.py index 0f2928f14..81af0f0e8 100644 --- a/examples/tutorials/series/_source/tutorials/t04_models.py +++ b/examples/tutorials/series/_source/tutorials/t04_models.py @@ -13,8 +13,8 @@ def cells(): "How to compare several models fairly on the same patients", "How to add a gradient-boosted tree baseline (XGBoost) to a PyHealth benchmark", ], - 20, - "Tutorial 03.", + 15, + "Tutorial 03. A GPU runtime adds the sequence models.", ), md(""" ## The models at a glance @@ -116,9 +116,11 @@ def __call__(self, patient): maximum epochs, the best epoch picked on validation PR-AUC, and the same test set. Only the model changes. - Sequence models are slow on a CPU with 35,000 training admissions, so - on a CPU runtime we train the three fastest; on a GPU runtime - (Runtime > Change runtime type) all five run in a few minutes. + Sequence models read every drug of every admission in order, which is + slow on a free CPU runtime (more than 30 minutes each for 35,000 + admissions). On a CPU runtime we therefore train the two pooled + models; on a GPU runtime (Runtime > Change runtime type > T4 GPU) + all five neural models run in a few minutes. """), code(""" import time @@ -150,12 +152,13 @@ def fit_and_test(name, model_class, **kwargs): return trainer - models = [("LogisticRegression", LogisticRegression), ("MLP", MLP), ("RNN", RNN)] + models = [("LogisticRegression", LogisticRegression), ("MLP", MLP)] if torch.cuda.is_available(): - models += [("Transformer", Transformer), ("RETAIN", RETAIN)] + models += [("RNN", RNN), ("Transformer", Transformer), ("RETAIN", RETAIN)] else: - print("CPU runtime: skipping Transformer and RETAIN (about 10 minutes each on CPU). " - "Switch to a GPU runtime to include them.") + print("CPU runtime: training LogisticRegression and MLP only. The sequence models " + "(RNN, Transformer, RETAIN) take 30+ minutes each on a free CPU runtime; " + "switch to a GPU runtime to include them.") for name, cls in models: fit_and_test(name, cls) """), From 3a16f8b34f50bf0c9ca88cc628411ed68ccbe50c Mon Sep 17 00:00:00 2001 From: solarsys Date: Fri, 9 Oct 2026 11:56:52 -0400 Subject: [PATCH 5/6] Tutorials 04, 07: wording and CPU budget from the Colab test run 07 trains one epoch on a CPU runtime (~10 minutes per epoch on Colab's free 2-CPU runtime) and recommends a GPU runtime up front; 04 no longer claims trees train much faster than the pooled neural models, which is not true on 2 CPUs. Co-Authored-By: Claude Opus 5.5 --- examples/tutorials/series/04_choosing_a_model.ipynb | 8 ++++---- examples/tutorials/series/07_clinical_text.ipynb | 10 +++++++--- .../tutorials/series/_source/tutorials/t04_models.py | 8 ++++---- .../tutorials/series/_source/tutorials/t07_text.py | 10 +++++++--- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/examples/tutorials/series/04_choosing_a_model.ipynb b/examples/tutorials/series/04_choosing_a_model.ipynb index 01b1fc908..c83e27641 100644 --- a/examples/tutorials/series/04_choosing_a_model.ipynb +++ b/examples/tutorials/series/04_choosing_a_model.ipynb @@ -90,8 +90,8 @@ "| `XGBoostModel` | Gradient-boosted trees on code counts or tabular features | Tabular or bag-of-codes data; the standard strong baseline |\n", "\n", "On tabular and bag-of-codes data, gradient-boosted trees are often as\n", - "good as neural models and much faster to train, so a benchmark without\n", - "them can overstate what a neural model adds.\n", + "good as neural models, and far cheaper than sequence models, so a\n", + "benchmark without them can overstate what a neural model adds.\n", "\n", "## The task: spotting heart failure from the medication list\n", "\n", @@ -334,8 +334,8 @@ "\n", "- Compare every model with the random-guess line first: PR-AUC is only\n", " meaningful relative to the prevalence.\n", - "- Look at cost as well as score: training time and model size differ\n", - " by orders of magnitude for similar accuracy.\n", + "- Look at cost as well as score: training time and model size vary a\n", + " lot between models with similar accuracy.\n", "- Small gaps can be noise. Before\n", " declaring a winner, compute bootstrap intervals of the paired\n", " difference (Tutorial 03, `paired_bootstrap_diff`).\n", diff --git a/examples/tutorials/series/07_clinical_text.ipynb b/examples/tutorials/series/07_clinical_text.ipynb index c8bc963aa..c1c1c7a54 100644 --- a/examples/tutorials/series/07_clinical_text.ipynb +++ b/examples/tutorials/series/07_clinical_text.ipynb @@ -84,8 +84,11 @@ "multilabel, binary).\n", "\n", "Model size matters on a free runtime. We pick automatically: a\n", - "clinical BERT (110M parameters, trained on MIMIC notes) on a GPU, or a\n", - "small general BERT (11M parameters) on a CPU." + "clinical BERT (110M parameters, trained on MIMIC notes) for 3 epochs on\n", + "a GPU, or a small general BERT (11M parameters) for 1 epoch on a CPU.\n", + "**Use a GPU runtime if you can** (Runtime > Change runtime type > T4\n", + "GPU): on a CPU even the small model needs about 10 minutes per epoch,\n", + "and scores after one epoch are lower." ] }, { @@ -100,7 +103,8 @@ "if torch.cuda.is_available():\n", " MODEL_NAME, EPOCHS = \"emilyalsentzer/Bio_ClinicalBERT\", 3\n", "else:\n", - " MODEL_NAME, EPOCHS = \"google/bert_uncased_L-4_H-256_A-4\", 2\n", + " # About 10 minutes per epoch on a free CPU runtime; a GPU is much faster.\n", + " MODEL_NAME, EPOCHS = \"google/bert_uncased_L-4_H-256_A-4\", 1\n", "print(\"device:\", \"cuda\" if torch.cuda.is_available() else \"cpu\", \"| model:\", MODEL_NAME)" ] }, diff --git a/examples/tutorials/series/_source/tutorials/t04_models.py b/examples/tutorials/series/_source/tutorials/t04_models.py index 81af0f0e8..0da82bcee 100644 --- a/examples/tutorials/series/_source/tutorials/t04_models.py +++ b/examples/tutorials/series/_source/tutorials/t04_models.py @@ -33,8 +33,8 @@ def cells(): | `XGBoostModel` | Gradient-boosted trees on code counts or tabular features | Tabular or bag-of-codes data; the standard strong baseline | On tabular and bag-of-codes data, gradient-boosted trees are often as - good as neural models and much faster to train, so a benchmark without - them can overstate what a neural model adds. + good as neural models, and far cheaper than sequence models, so a + benchmark without them can overstate what a neural model adds. ## The task: spotting heart failure from the medication list @@ -217,8 +217,8 @@ def fit_and_test(name, model_class, **kwargs): - Compare every model with the random-guess line first: PR-AUC is only meaningful relative to the prevalence. - - Look at cost as well as score: training time and model size differ - by orders of magnitude for similar accuracy. + - Look at cost as well as score: training time and model size vary a + lot between models with similar accuracy. - Small gaps can be noise. Before declaring a winner, compute bootstrap intervals of the paired difference (Tutorial 03, `paired_bootstrap_diff`). diff --git a/examples/tutorials/series/_source/tutorials/t07_text.py b/examples/tutorials/series/_source/tutorials/t07_text.py index 74d317bb3..b87cf6bbe 100644 --- a/examples/tutorials/series/_source/tutorials/t07_text.py +++ b/examples/tutorials/series/_source/tutorials/t07_text.py @@ -27,8 +27,11 @@ def cells(): multilabel, binary). Model size matters on a free runtime. We pick automatically: a - clinical BERT (110M parameters, trained on MIMIC notes) on a GPU, or a - small general BERT (11M parameters) on a CPU. + clinical BERT (110M parameters, trained on MIMIC notes) for 3 epochs on + a GPU, or a small general BERT (11M parameters) for 1 epoch on a CPU. + **Use a GPU runtime if you can** (Runtime > Change runtime type > T4 + GPU): on a CPU even the small model needs about 10 minutes per epoch, + and scores after one epoch are lower. """), code(""" import torch @@ -36,7 +39,8 @@ def cells(): if torch.cuda.is_available(): MODEL_NAME, EPOCHS = "emilyalsentzer/Bio_ClinicalBERT", 3 else: - MODEL_NAME, EPOCHS = "google/bert_uncased_L-4_H-256_A-4", 2 + # About 10 minutes per epoch on a free CPU runtime; a GPU is much faster. + MODEL_NAME, EPOCHS = "google/bert_uncased_L-4_H-256_A-4", 1 print("device:", "cuda" if torch.cuda.is_available() else "cpu", "| model:", MODEL_NAME) """), md(""" From 445a3a6671f1e0b5265deb8cfb12b08789458c91 Mon Sep 17 00:00:00 2001 From: solarsys Date: Fri, 9 Oct 2026 12:20:43 -0400 Subject: [PATCH 6/6] Tutorial 08: print the created files instead of a character count Co-Authored-By: Claude Opus 5.5 --- examples/tutorials/series/08_contributing.ipynb | 3 ++- .../tutorials/series/_source/tutorials/t08_contributing.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/tutorials/series/08_contributing.ipynb b/examples/tutorials/series/08_contributing.ipynb index 371ab0e46..ab2e8908f 100644 --- a/examples/tutorials/series/08_contributing.ipynb +++ b/examples/tutorials/series/08_contributing.ipynb @@ -141,7 +141,8 @@ " timestamp: \"visit_time\"\n", " timestamp_format: \"%Y-%m-%d %H:%M\"\n", " attributes: [\"diagnosis\", \"systolic_bp\"]\n", - "''')" + "''')\n", + "print(sorted(p.name for p in root.iterdir()))" ] }, { diff --git a/examples/tutorials/series/_source/tutorials/t08_contributing.py b/examples/tutorials/series/_source/tutorials/t08_contributing.py index ddf218f09..c9a22dff2 100644 --- a/examples/tutorials/series/_source/tutorials/t08_contributing.py +++ b/examples/tutorials/series/_source/tutorials/t08_contributing.py @@ -78,6 +78,7 @@ def cells(): timestamp_format: "%Y-%m-%d %H:%M" attributes: ["diagnosis", "systolic_bp"] ''') + print(sorted(p.name for p in root.iterdir())) """), md(""" Notice the raw data has inconsistent code case (`i10` vs `I10`). The