def worker(output, worker_id):
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
config = json.loads((output / "config.json").read_text())
prompt = (output / "prompt.txt").read_text()
(output / f"worker-{worker_id}").mkdir(exist_ok=True)
states = load_states(output, worker_id, config)
device = f"cuda:{config['gpu_ids'][worker_id]}"
tokenizer = AutoTokenizer.from_pretrained(REPOSITORY / config["model"], padding_side="left")
model = AutoModelForCausalLM.from_pretrained(REPOSITORY / config["model"], dtype=torch.bfloat16,
attn_implementation="sdpa").to(device).eval()
def request(state, targets, round_number):
messages = _messages(state["row"], targets, round_number, state["latest"], config["primary_attempts"], prompt)
text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, enable_thinking=False)
return {"state": state, "targets": targets, "text": text, "length": len(tokenizer.encode(text))}
def generate(batch, round_number):
inputs = tokenizer([r["text"] for r in batch], padding=True, return_tensors="pt").to(device)
options = {}
# After a malformed answer, constrain the JSON structure of the retry.
if round_number and any(r["state"]["last"]["error"] for r in batch):
options["prefix_allowed_tokens_fn"] = json_constraint(tokenizer, [r["targets"] for r in batch])
cap = min(4096, max(512, 128 + 96 * max(len(r["targets"]) for r in batch)))
try:
with torch.inference_mode():
output_ids = model.generate(**inputs, do_sample=False, max_new_tokens=cap,
pad_token_id=tokenizer.pad_token_id, **options)
except torch.OutOfMemoryError:
if len(batch) == 1:
raise
torch.cuda.empty_cache()
generate(batch[:len(batch) // 2], round_number)
generate(batch[len(batch) // 2:], round_number)
return
attempts = []
for item, tokens in zip(batch, output_ids[:, inputs.input_ids.shape[1]:]):
raw = tokenizer.decode(tokens, skip_special_tokens=True)
summaries, error = parse(raw, item["targets"], tokenizer.eos_token_id in tokens.tolist())
attempts.append({"news_id": item["state"]["row"]["news_id"], "round": round_number,
"targets": item["targets"], "summaries": summaries, "error": error})
record(item["state"], attempts[-1], config)
write_jsonl(output / f"worker-{worker_id}/attempts.jsonl", attempts, mode="a")
for round_number in range(config["max_attempts"]):
recovery = round_number >= config["primary_attempts"]
pending = []
for state in states.values():
if missing(state) and (recovery or state["round"] <= round_number):
groups = [[sid] for sid in missing(state)] if recovery else [missing(state)]
pending += [request(state, group, round_number) for group in groups
if (round_number, tuple(group)) not in state["seen"]]
pending.sort(key=lambda r: (r["length"], r["state"]["row"]["news_id"], r["targets"]))
position = 0
while position < len(pending):
batch = pending[position:position + batch_size_for(pending[position]["length"], len(pending[position]["targets"]))]
while len(batch) > 1 and batch[-1]["length"] > max(1.35 * batch[0]["length"], batch[0]["length"] + 512):
batch = batch[:-1]
generate(batch, round_number)
position += len(batch)
unresolved = sum(bool(missing(state)) for state in states.values())
if unresolved:
raise RuntimeError(f"{unresolved} articles still have segments without a valid summary")