PhenoBERT: official API loading and offset-preserving parsing, and fine-tuning of its three-way HPO phrase matcher.
python -m knowledge_graph.clinical.phenobert # fine-tune on the training notes, score the test notes
full_unpickling()
torch.load with weights_only=False while PhenoBERT runs: its released models and the Stanza 1.1 resources it
loads on demand are pickled objects, which PyTorch 2.6 refuses by default.
Source code in src/knowledge_graph/clinical/phenobert.py
25
26
27
28
29
30
31
32
33
34
35
36 | @contextmanager
def full_unpickling():
"""torch.load with weights_only=False while PhenoBERT runs: its released models and the Stanza 1.1 resources it
loads on demand are pickled objects, which PyTorch 2.6 refuses by default."""
import torch
load = torch.load
torch.load = functools.partial(load, weights_only=False)
try:
yield
finally:
torch.load = load
|
PhenoBERTBackend
Own a PhenoBERT model and reuse its classifiers across predictions.
Source code in src/knowledge_graph/clinical/phenobert.py
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62 | class PhenoBERTBackend:
"""Own a PhenoBERT model and reuse its classifiers across predictions."""
def __init__(self) -> None:
import nltk
import stanza
from stanza.resources.common import DEFAULT_MODEL_DIR
nltk.data.path.insert(0, str(DATA / "resources" / "nltk")) # stopwords, wordnet, and the POS tagger
if not (Path(DEFAULT_MODEL_DIR) / "en" / "ner" / "i2b2.pt").exists(): # Stanza 1.1 does not download by itself
stanza.download("en", package="mimic", processors={"ner": "i2b2"}, verbose=False)
if str(PHENOBERT) not in sys.path:
sys.path.insert(0, str(PHENOBERT))
with full_unpickling():
self.api = importlib.import_module("api").PhenoBERTAPI()
self.bert_model, self.hpo_tree = self.api.bert_model, self.api.hpo_tree
def annotate_text(self, text: str) -> str:
with full_unpickling():
return self.api.annotate_text(text)
def get_most_related_HPO_term(self, phrases: list[str]) -> list:
with full_unpickling():
return self.api.get_most_related_HPO_term(phrases)
|
api = importlib.import_module('api').PhenoBERTAPI()
instance-attribute
annotate_text(text: str) -> str
Source code in src/knowledge_graph/clinical/phenobert.py
| def annotate_text(self, text: str) -> str:
with full_unpickling():
return self.api.annotate_text(text)
|
Source code in src/knowledge_graph/clinical/phenobert.py
| def get_most_related_HPO_term(self, phrases: list[str]) -> list:
with full_unpickling():
return self.api.get_most_related_HPO_term(phrases)
|
normalised_offset_map(text)
PhenoBERT's normalised text, which drops hyphens between an uppercase letter and a digit, and a map from its
character boundaries to those of text.
Source code in src/knowledge_graph/clinical/phenobert.py
65
66
67
68
69
70
71
72
73
74
75
76 | def normalised_offset_map(text):
"""PhenoBERT's normalised text, which drops hyphens between an uppercase letter and a digit, and a map from its
character boundaries to those of `text`."""
chars, boundaries = [], [0]
deleted = {match.start() for match in re.finditer(r"(?<=[A-Z])-(?=[\d])", text)}
for index, char in enumerate(text):
if index in deleted:
boundaries[-1] = index + 1
continue
chars.append(char)
boundaries.append(index + 1)
return "".join(chars), boundaries
|
parse_predictions(raw_output, segment, source_text)
Mentions of PhenoBERT's tab-separated output at source-note offsets.
Source code in src/knowledge_graph/clinical/phenobert.py
79
80
81
82
83
84
85
86
87
88
89
90
91
92 | def parse_predictions(raw_output, segment, source_text):
"""Mentions of PhenoBERT's tab-separated output at source-note offsets."""
normalized, boundary_map = normalised_offset_map(segment["text"])
rows, remaining = [], raw_output
while remaining:
start, stop, remaining = remaining.split("\t", 2)
local_begin, local_end = int(start), int(stop)
# The mention text may contain tabs or newlines; its offsets give its length.
trailer, _, remaining = remaining[local_end - local_begin + 1:].partition("\n")
fields = trailer.split("\t")
begin, end = segment["begin"] + boundary_map[local_begin], segment["begin"] + boundary_map[local_end]
rows.append({"begin": begin, "end": end, "text": source_text[begin:end], "hpo_id": fields[0],
"score": float(fields[1]), "negative": len(fields) == 3, "segment_id": segment["segment_id"]})
return rows
|
pairs(ids, ontology)
Build phrase pairs; conflicting inputs prefer same concept (2), parent (1), then unrelated (0).
Source code in src/knowledge_graph/clinical/phenobert.py
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135 | def pairs(ids, ontology):
"""Build phrase pairs; conflicting inputs prefer same concept (2), parent (1), then unrelated (0)."""
rng = random.Random(42)
keys = sorted(ontology.terms)
ancestors = {}
def related(a, b):
for key in (a, b):
if key not in ancestors:
ancestors[key] = ontology.ancestors(key)
return b in ancestors[a] or a in ancestors[b]
output = set()
for note_id in ids:
for entity in load_json(CORPUS / "hpo_adjudication/review_final" / f"{note_id}.json")["symptom_entities"]:
if entity["hpo_status"] != "linked":
continue
key = ontology.canonical_id(entity["hpo_id"])
term = ontology.terms[key]
for mention in entity["mentions"]:
phrase = " ".join(mention["text"].split())
output.add((phrase, term.label, 2))
parents = [p for p in ontology.parents.get(key, []) if p in ontology.terms]
if parents:
output.add((phrase, ontology.terms[parents[0]].label, 1))
negative = rng.choice(keys)
while negative == key or related(key, negative):
negative = rng.choice(keys)
output.add((phrase, ontology.terms[negative].label, 0))
# Match addWords: lowercase, replace ASCII punctuation, and split whitespace.
spaces = str.maketrans({char: " " for char in punctuation})
def input_key(row):
fields = (row[0] + "::" + row[1]).lower().split("::")[:2]
return tuple(tuple(field.translate(spaces).split()) for field in fields)
targets = {}
for row in output:
key = input_key(row)
targets[key] = max(targets.get(key, 0), row[2])
return sorted(row for row in output if row[2] == targets[input_key(row)])
|
dataset(rows)
Source code in src/knowledge_graph/clinical/phenobert.py
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154 | def dataset(rows):
from fastNLP import DataSet
from my_bert_match import addSeqlen, addWordPiece, addWords, processItem, processNum
d = DataSet({"raw_words": [a + "::" + b for a, b, c in rows], "target": [c for a, b, c in rows]})
for func, name in [
(addWords, "p_words"),
(addWordPiece, "t_words"),
(processItem, "word_pieces"),
(processNum, "word_nums"),
(addSeqlen, "seq_len"),
]:
d.apply(func, new_field_name=name)
for k in ["word_pieces", "word_nums", "seq_len"]:
d.field_arrays[k].is_input = True
d.field_arrays["target"].is_target = True
return d
|
fit_matcher(api, train_rows, valid_rows, out)
Fine-tune on training pairs and restore the checkpoint selected by validation macro F1.
Source code in src/knowledge_graph/clinical/phenobert.py
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191 | def fit_matcher(api, train_rows, valid_rows, out):
"""Fine-tune on training pairs and restore the checkpoint selected by validation macro F1."""
import torch
from fastNLP import DataSetIter, RandomSampler # the vendored copy is importable once PhenoBERTBackend is loaded
from fastNLP.core.utils import _move_dict_value_to_device
train = dataset(train_rows)
valid = dataset(valid_rows)
model = api.bert_model.to("cuda")
optimizer = torch.optim.Adam(model.parameters(), lr=2e-5)
selection = EarlyStopping()
for _ in range(10):
model.train()
for x, y in DataSetIter(batch_size=8, dataset=train, sampler=RandomSampler()):
_move_dict_value_to_device(x, y, device=torch.device("cuda"))
optimizer.zero_grad(set_to_none=True)
loss = torch.nn.functional.cross_entropy(
model.forward(x["word_pieces"], x["word_nums"], x["seq_len"])["pred"], y["target"])
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
model.eval()
predicted, gold = [], []
with torch.no_grad():
for x, y in DataSetIter(batch_size=8, dataset=valid, sampler=None):
_move_dict_value_to_device(x, y, device=torch.device("cuda"))
predicted.extend(model.forward(x["word_pieces"], x["word_nums"], x["seq_len"])["pred"].argmax(-1).cpu().tolist())
gold.extend(y["target"].cpu().tolist())
score = sum(metric({i for i, v in enumerate(gold) if v == k}, {i for i, v in enumerate(predicted) if v == k})["f1"]
for k in range(3)) / 3
if selection.update(score, lambda: torch.save(model.state_dict(), out / "best_state.pt")):
break
model.load_state_dict(torch.load(out / "best_state.pt"))
model.eval()
|
evaluate(api, ontology, ids, out)
NER and HPO-link scores of the test notes under every condition.
Source code in src/knowledge_graph/clinical/phenobert.py
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211 | def evaluate(api, ontology, ids, out):
"""NER and HPO-link scores of the test notes under every condition."""
from knowledge_graph.clinical.hpo import gold_links
docs = documents(ids)
nlp = segmenter()
links_gold = gold_links(ontology, ids)
results = {}
for condition in CONDITIONS:
rows = {document["id"]: [row for segment in segments(nlp, document["text"], document["reference_segments"], condition)
for row in parse_predictions(api.annotate_text(segment["text"]), segment, document["text"])]
for document in docs}
links = {(note_id, r["begin"], r["end"], hpo_id) for note_id, found in rows.items() for r in found
if (hpo_id := ontology.canonical_id(r.get("hpo_id")))}
results[condition] = {"ner": score_ner(docs, rows), "el": metric(links_gold, links)}
write_json(out / f"test_{condition}.json", {"metrics": results[condition],
"documents": [{"id": n, "predictions": p} for n, p in rows.items()]})
write_json(out / "test_results.json", results)
|
main()
Source code in src/knowledge_graph/clinical/phenobert.py
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238 | def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--out", type=Path, default=CLINICAL_RUNS / "phenobert")
parser.add_argument("--evaluate-only", action="store_true", help="Evaluate best_state.pt without training.")
args = parser.parse_args()
import numpy as np
import torch
from knowledge_graph.clinical.hpo import HpoOntology
random.seed(42)
np.random.seed(42)
torch.manual_seed(42)
torch.set_num_threads(4)
out = args.out.resolve()
out.mkdir(parents=True, exist_ok=True)
folds = split()
ontology = HpoOntology()
api = PhenoBERTBackend()
if args.evaluate_only:
api.bert_model.load_state_dict(torch.load(out / "best_state.pt", map_location="cpu", weights_only=True))
api.bert_model.eval()
else:
fit_matcher(api, pairs(folds["train_ids"], ontology), pairs(folds["validation_ids"], ontology), out)
evaluate(api, ontology, folds["test_ids"], out)
|