Blocks and heading sections of MIMIC discharge notes, split into sentences.
python -m topic_segmentation.data.mimic.segments --cache NOTES_by_id.json [--cache MORE_by_id.json] --out SEGMENTS.json
Blocks are separated by blank lines; within a block, hash, colon, underlined, and standalone headings open sections
(data.mimic.rules). Lines before the first heading form the block prelude.
start_section(sections: list[dict[str, Any]], current: dict[str, Any] | None, label: str) -> dict[str, Any]
Source code in src/topic_segmentation/data/mimic/segments.py
28
29
30
31
32
33
34
35
36
37
38 | def start_section(
sections: list[dict[str, Any]],
current: dict[str, Any] | None,
label: str,
) -> dict[str, Any]:
if current is not None:
sections.append(current)
return {
"label": label,
"_lines": [],
}
|
segment_block(block_lines: list[tuple[int, str]]) -> dict[str, Any]
Source code in src/topic_segmentation/data/mimic/segments.py
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91 | def segment_block(block_lines: list[tuple[int, str]]) -> dict[str, Any]:
prelude_lines: list[str] = []
sections: list[dict[str, Any]] = []
current: dict[str, Any] | None = None
i = 0
while i < len(block_lines):
_, line = block_lines[i]
if i + 1 < len(block_lines):
underline_label = underline_heading_label(line, block_lines[i + 1][1])
if underline_label is not None:
current = start_section(sections, current, underline_label)
i += 2
continue
header, header_type = hash_header_parts(line), "hash_heading"
if header is None:
header = colon_header_parts(line)
header_type = "major_inline_heading" if header is not None and header[1] else "colon_heading"
if header is not None:
label, value = header
# A repeated heading right after the same heading continues that section.
repeated = current is not None and label_key(label) == label_key(current["label"]) and not current["_lines"]
if not repeated:
current = start_section(sections, current, label)
if value:
current["_lines"].append(value)
underlined = (not repeated and header_type != "hash_heading" and i + 1 < len(block_lines)
and UNDERLINE_RE.match(block_lines[i + 1][1]))
i += 2 if underlined else 1
continue
standalone_header = standalone_header_parts(line)
if standalone_header is not None:
label, _ = standalone_header
current = start_section(sections, current, label)
i += 1
continue
if current is None:
prelude_lines.append(line)
else:
current["_lines"].append(line)
i += 1
if current is not None:
sections.append(current)
sections = [{"sentences": split_sentences(section["_lines"])} for section in sections]
prelude = split_sentences(prelude_lines)
return {"prelude": {"sentences": prelude} if prelude else None, "sections": sections}
|
build_note_segments(note)
Normalize front matter and split a note into blocks, preludes, and heading sections.
Source code in src/topic_segmentation/data/mimic/segments.py
| def build_note_segments(note):
"""Normalize front matter and split a note into blocks, preludes, and heading sections."""
note = normalize_front_matter_doc(note)
blocks = [segment_block(lines) for _, _, lines in split_blocks(note["text"])]
return {"note_id": note["note_id"], "table": note["table"], "blocks": blocks}
|
main()
Source code in src/topic_segmentation/data/mimic/segments.py
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116 | def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--cache", type=Path, action="append", required=True,
help="Note cache (*_by_id.json) of data.mimic.sample; repeat for several")
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
order, notes = {}, {}
for path in args.cache:
cache = json.loads(path.read_text(encoding="utf-8"))
for note_id in cache["order"]:
if note_id in notes:
raise ValueError(f"note {note_id} is in two caches")
notes[note_id] = build_note_segments(cache["notes"][note_id])
order.setdefault(notes[note_id]["table"], []).append(note_id)
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(json.dumps({"order": order, "notes": notes}, ensure_ascii=False, indent=2), encoding="utf-8")
|