Skip to content

relations

knowledge_graph.news.relations

Predict and score DWIE relations with ATLOP or DREEAM, using reference or predicted entities.

score_key(row: dict) -> tuple

Set key for scoring: unmapped relations never match a gold relation.

Source code in src/knowledge_graph/news/relations.py
20
21
22
23
def score_key(row: dict) -> tuple:
    """Set key for scoring: unmapped relations never match a gold relation."""
    key = (row["document_id"], row["subject"], row["relation"], row["object"])
    return key if row.get("mapped", True) else (row["document_id"], "unmapped", *key[1:])

to_gold(local: set[tuple], mappings: list[dict]) -> list[dict]

Relations between gold concepts, once per document; a predicted-mention relation with an endpoint outside the gold mentions keeps its prediction ids and is marked unmapped.

Source code in src/knowledge_graph/news/relations.py
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
def to_gold(local: set[tuple], mappings: list[dict]) -> list[dict]:
    """Relations between gold concepts, once per document; a predicted-mention relation with an endpoint outside the
    gold mentions keeps its prediction ids and is marked unmapped."""
    by_title = {mapping["title"]: mapping for mapping in mappings}
    rows = {}
    for title, head, relation, tail in local:
        mapping = by_title[title]
        gold = mapping["local_to_gold"]
        mapped = gold[head] is not None and gold[tail] is not None
        ids = gold if mapped else mapping["local_to_prediction"]
        row = {"document_id": mapping["document_id"], "subject": ids[head], "relation": relation, "object": ids[tail]}
        if not mapped:
            row["mapped"] = False
        rows[score_key(row)] = row
    return sorted(rows.values(), key=lambda row: (not row.get("mapped", True), score_key(row)))

evaluate(predictions: dict[str, list[dict]], documents: list[dict], groups: dict[str, set[str]]) -> dict

Scores of every condition on each document group, and the document-level row when both of its parts exist.

Source code in src/knowledge_graph/news/relations.py
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
def evaluate(predictions: dict[str, list[dict]], documents: list[dict], groups: dict[str, set[str]]) -> dict:
    """Scores of every condition on each document group, and the document-level row when both of its parts exist."""
    gold = observable_relations(documents)
    results = {}
    for condition, rows in predictions.items():
        found = {score_key(row) for row in rows}
        results[condition] = {
            group: precision_recall_f1({r for r in gold if r[0] in ids}, {r for r in found if r[0] in ids})
            for group, ids in groups.items()
        }
    if ENCODER_WINDOWS in results and "chunks1024" in results:
        long, short = results[ENCODER_WINDOWS]["long"], results["chunks1024"]["short"]
        tp, predicted = long["tp"] + short["tp"], long["predicted"] + short["predicted"]
        results["document_level"] = {"all": count_metrics(tp, len(gold), predicted), "long": long, "short": short}
    return results

write_scores(predictions_path, gold_path, out, tokenizer)

Source code in src/knowledge_graph/news/relations.py
60
61
62
def write_scores(predictions_path, gold_path, out, tokenizer):
    documents = load_json(gold_path)["documents"]
    write_json(out, evaluate(load_json(predictions_path), documents, document_groups(documents, tokenizer)), indent=2)

score(args: argparse.Namespace) -> None

Source code in src/knowledge_graph/news/relations.py
65
66
def score(args: argparse.Namespace) -> None:
    write_scores(args.predictions, args.gold, args.out, AutoTokenizer.from_pretrained(ENCODER))

predict(args: argparse.Namespace) -> None

Write predictions and scores for the selected input conditions.

Source code in src/knowledge_graph/news/relations.py
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
def predict(args: argparse.Namespace) -> None:
    """Write predictions and scores for the selected input conditions."""
    model = DocREPredictor(args.implementation, args.inputs / "rel2id.json", args.model)
    mappings = load_json(args.inputs / "mappings.json")
    predictions = {}
    for name in CONDITIONS:
        if f"test_{name}.json" in mappings:  # predicted-mention inputs cover fewer conditions
            documents = load_json(args.inputs / f"test_{name}.json")
            if name == "document":  # whole documents read through overlapping encoder windows
                predictions[ENCODER_WINDOWS] = to_gold(model.predict_with_windows(documents), mappings[f"test_{name}.json"])
            else:
                predictions[name] = to_gold(model.predict(documents), mappings[f"test_{name}.json"])
    path = args.out_dir / "predictions.json"
    write_json(path, predictions)
    write_scores(path, args.gold, args.out_dir / "results.json", model.tokenizer)

main() -> None

Source code in src/knowledge_graph/news/relations.py
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    sub = parser.add_subparsers(dest="command", required=True)
    p = sub.add_parser("predict", help="Run a trained model on DocRED inputs and score its relations.")
    p.add_argument("--implementation", choices=list(IMPLEMENTATIONS), required=True)
    p.add_argument("--inputs", type=Path, required=True, help="Gold-entity or predicted-mention DocRED inputs.")
    p.add_argument("--model", type=Path, required=True, help="best.pt of the implementation.")
    p.add_argument("--gold", type=Path, required=True, help="dwie_test_gold.json")
    p.add_argument("--out-dir", type=Path, required=True)
    p.set_defaults(run=predict)
    p = sub.add_parser("score", help="Score saved predictions.")
    p.add_argument("--gold", type=Path, required=True, help="dwie_test_gold.json")
    p.add_argument("--predictions", type=Path, required=True)
    p.add_argument("--out", type=Path, required=True)
    p.set_defaults(run=score)
    args = parser.parse_args()
    args.run(args)