Skip to content

sample

topic_segmentation.data.mimic.sample

Sample MIMIC-IV discharge notes that contain a hospital course section.

python -m topic_segmentation.data.mimic.sample --seed 42 --n 1000 --out-dir OUT [--exclude-sample-json EARLIER.json ...]

Reservoir sampling over discharge.csv.gz; notes and patients of earlier samples are excluded. Writes OUT/discharge_random<n>.json (note and patient IDs) and OUT/discharge_random<n>_by_id.json (the notes).

excluded_ids(paths)

Collect note and patient IDs from earlier samples.

Source code in src/topic_segmentation/data/mimic/sample.py
24
25
26
27
28
29
30
31
def excluded_ids(paths):
    """Collect note and patient IDs from earlier samples."""
    notes, subjects = set(), set()
    for path in paths:
        for row in json.loads(Path(path).read_text(encoding="utf-8"))["rows"]:
            notes.add(row["note_id"])
            subjects.add(row["subject_id"])
    return notes, subjects

reservoir_sample(size, rng, excluded_notes, excluded_subjects)

Return a uniform sample sorted by note ID.

Source code in src/topic_segmentation/data/mimic/sample.py
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
def reservoir_sample(size, rng, excluded_notes, excluded_subjects):
    """Return a uniform sample sorted by note ID."""
    sample, eligible = [], 0
    with gzip.open(NOTES, "rt", encoding="utf-8", newline="") as handle:
        for row in csv.DictReader(handle):
            if not COURSE_HEADING_RE.search(row["text"]):
                continue
            if row["note_id"] in excluded_notes or row["subject_id"] in excluded_subjects:
                continue
            eligible += 1
            slot = len(sample)
            if slot >= size:
                slot = rng.randrange(eligible)
                if slot >= size:
                    continue
            item = {"table": "discharge", "note_id": row["note_id"],
                    "subject_id": row["subject_id"], "text": row["text"]}
            if slot == len(sample):
                sample.append(item)
            else:
                sample[slot] = item
    sample.sort(key=lambda note: note["note_id"])
    return sample

main()

Source code in src/topic_segmentation/data/mimic/sample.py
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
def main():
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("--seed", type=int, required=True)
    parser.add_argument("--n", type=int, required=True)
    parser.add_argument("--out-dir", type=Path, required=True)
    parser.add_argument("--exclude-sample-json", type=Path, action="append", default=[],
                        help="Earlier sample whose notes and patients are excluded; may be repeated")
    args = parser.parse_args()

    name = f"discharge_random{args.n}"
    notes = reservoir_sample(args.n, random.Random(args.seed), *excluded_ids(args.exclude_sample_json))
    if len(notes) != args.n:
        raise RuntimeError(f"Requested {args.n} notes but only {len(notes)} were eligible")
    args.out_dir.mkdir(parents=True, exist_ok=True)
    summary = {"rows": [{"note_id": note["note_id"], "subject_id": note["subject_id"]} for note in notes]}
    (args.out_dir / f"{name}.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
    cache = {"order": [note["note_id"] for note in notes], "notes": {note["note_id"]: note for note in notes}}
    (args.out_dir / f"{name}_by_id.json").write_text(json.dumps(cache, ensure_ascii=False), encoding="utf-8")