Filter MIND-large to articles with full bodies, split its impressions by day, and pack the three representations.
python -m news_recommendation.preparation all --output runs/data
split filters articles and impressions, reserving the last two days for validation and test.
inputs packs each title with an abstract, segment summaries, or an article summary into 512 BERT tokens.
Segment summaries are dropped whole when they do not fit: by the saved selection when replaying,
otherwise at random with a per-article seed. See README.md for filtering criteria.
read_jsonl(paths)
Source code in src/news_recommendation/preparation.py
| def read_jsonl(paths):
return {row["news_id"]: row for path in paths for row in map(json.loads, path.open())}
|
select_news(paths, tokenizer)
MIND-large train and dev news to keep, and the matched body of each.
Source code in src/news_recommendation/preparation.py
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53 | def select_news(paths, tokenizer):
"""MIND-large train and dev news to keep, and the matched body of each."""
news = {}
for split in ("train", "dev"):
with (paths["mind"] / split / "news.tsv").open() as stream:
news |= {fields[0]: fields for fields in (line.rstrip("\r\n").split("\t") for line in stream)}
by_url = {}
for row in read_jsonl([paths["segmented"]]).values():
by_url.setdefault(row["url"].strip(), []).append(row)
candidates = {}
for key, fields in news.items():
matches = by_url.get(fields[5].strip(), []) if fields[5].strip() else []
if len(matches) == 1 and 0 < len(matches[0]["segments"]) <= MAX_SEGMENTS and fields[4].strip():
candidates[key] = matches[0]
keys = sorted(candidates)
lengths = [len(ids) for start in range(0, len(keys), 1000) for ids in tokenizer(
[candidates[key]["full_body"].lower() for key in keys[start:start + 1000]], add_special_tokens=False)["input_ids"]]
kept = {key: {"news_id": key, "body_source_id": candidates[key]["news_id"], "body_bert_tokens": length,
"body_sections": len(candidates[key]["segments"])}
for key, length in zip(keys, lengths) if length >= MIN_BODY_TOKENS}
return {key: news[key] for key in kept}, kept
|
read_impressions(paths, news)
Train then dev impressions restricted to news, renumbered from 1.
Source code in src/news_recommendation/preparation.py
56
57
58
59
60
61
62
63
64
65
66
67
68 | def read_impressions(paths, news):
"""Train then dev impressions restricted to `news`, renumbered from 1."""
records = []
for split in ("train", "dev"):
with (paths["mind"] / split / "behaviors.tsv").open("rb") as stream:
for line in stream:
_, user, timestamp, history, candidates = line.decode().rstrip("\r\n").split("\t")
history = [key for key in history.split() if key in news]
candidates = [item for item in candidates.split() if item.rsplit("-", 1)[0] in news]
if (len(history) >= MIN_HISTORY and len(candidates) >= MIN_CANDIDATES
and {item[-1] for item in candidates} == {"0", "1"}):
records.append([user, timestamp, " ".join(history), " ".join(candidates)])
return ["\t".join([str(i), *record]) + "\n" for i, record in enumerate(records, 1)]
|
split_by_day(records)
Training, validation, and test records: the last day is test, the day before validation.
Source code in src/news_recommendation/preparation.py
| def split_by_day(records):
"""Training, validation, and test records: the last day is test, the day before validation."""
days = [datetime.strptime(record.split("\t")[2], "%m/%d/%Y %I:%M:%S %p").date() for record in records]
validation_day = max(days) - timedelta(days=1)
return {"train": [r for r, day in zip(records, days) if day < validation_day],
"validation": [r for r, day in zip(records, days) if day == validation_day],
"test": [r for r, day in zip(records, days) if day > validation_day]}
|
prepare_split(destination, paths)
Source code in src/news_recommendation/preparation.py
80
81
82
83
84
85
86
87
88
89 | def prepare_split(destination, paths):
news, metadata = select_news(paths, BertTokenizerFast.from_pretrained(paths["tokenizer"]))
destination.mkdir(parents=True)
(destination / "news.tsv").write_text("".join("\t".join(news[key]) + "\n" for key in sorted(news)))
with (destination / "news_body_metadata.csv").open("w", newline="") as stream:
writer = csv.DictWriter(stream, fieldnames=["news_id", "body_source_id", "body_bert_tokens", "body_sections"])
writer.writeheader()
writer.writerows(metadata[key] for key in sorted(metadata))
for name, records in split_by_day(read_impressions(paths, news)).items():
(destination / f"{name}.tsv").write_text("".join(records))
|
pack_text_input(tokenizer, title, text, length=512)
[CLS] title [SEP] text [SEP], cutting the end of the text to fit.
Source code in src/news_recommendation/preparation.py
| def pack_text_input(tokenizer, title, text, length=512):
"""[CLS] title [SEP] text [SEP], cutting the end of the text to fit."""
ids = [tokenizer.cls_token_id] + tokenizer.encode(title.lower(), add_special_tokens=False)[:length - 3]
ids.append(tokenizer.sep_token_id)
return ids + tokenizer.encode(text.lower(), add_special_tokens=False)[:length - len(ids) - 1] + [tokenizer.sep_token_id]
|
[CLS] title [SEP] summary [SEP] ... with whole summaries dropped to fit; returns the ids and kept segment IDs.
Source code in src/news_recommendation/preparation.py
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113 | def pack_section_input(tokenizer, title, summaries, news_id, length=512, selected_segment_ids=None):
"""[CLS] title [SEP] summary [SEP] ... with whole summaries dropped to fit; returns the ids and kept segment IDs."""
ids = [tokenizer.cls_token_id] + tokenizer.encode(title.lower(), add_special_tokens=False)[:length - 2]
ids.append(tokenizer.sep_token_id)
sections = [(row["segment_id"], tokenizer.encode(row["summary"].lower(), add_special_tokens=False))
for row in summaries]
if selected_segment_ids is None:
rng = random.Random(int.from_bytes(f"{SELECTION_SEED}:{news_id}".encode(), "big"))
while len(ids) + sum(len(tokens) + 1 for _, tokens in sections) > length:
sections.pop(rng.randrange(len(sections)))
else:
sections = [(sid, tokens) for sid, tokens in sections if sid in selected_segment_ids]
for _, tokens in sections:
ids += tokens + [tokenizer.sep_token_id]
return ids, [sid for sid, _ in sections]
|
inputs//features.npz for each representation, and the segment selection of the summary variant.
Source code in src/news_recommendation/preparation.py
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148 | def prepare_inputs(destination, paths):
"""inputs/<variant>/features.npz for each representation, and the segment selection of the summary variant."""
tokenizer = BertTokenizerFast.from_pretrained(paths["tokenizer"])
with (destination / "news.tsv").open() as stream:
news = {fields[0]: fields for fields in (line.rstrip("\n").split("\t") for line in stream)}
with (destination / "news_body_metadata.csv").open(newline="") as stream:
sources = {row["news_id"]: row["body_source_id"] for row in csv.DictReader(stream)}
sections = read_jsonl([paths["sections"]])
articles = read_jsonl(sorted(paths["articles"].glob("*/outputs.jsonl")))
saved = ({row["news_id"]: row["selected_segment_ids"] for row in map(json.loads, paths["selection"].open())}
if "selection" in paths else {})
keys = sorted(news)
for variant in VARIANTS:
folder = destination / "inputs" / variant
folder.mkdir(parents=True)
# Token ids, token types (all zero), and attention mask, side by side.
packed = np.zeros((len(keys), 3 * 512), dtype=np.int32)
selections = []
for position, key in enumerate(keys):
title, abstract, source = news[key][3], news[key][4], sources[key]
if variant == "title_section_summary_512":
ids, selected = pack_section_input(tokenizer, title, sections[source]["parsed"]["summaries"], key,
selected_segment_ids=saved.get(key))
selections.append({"news_id": key, "selected_segment_ids": selected})
else:
text = abstract if variant == "title_abstract_512" else articles[source]["summary"]
ids = pack_text_input(tokenizer, title, text)
packed[position, :512] = tokenizer.pad_token_id
packed[position, :len(ids)] = ids
packed[position, 1024:1024 + len(ids)] = 1
np.savez(folder / "features.npz", keys=np.asarray(keys), packed=packed)
if selections:
(folder / "selection.jsonl").write_text("".join(json.dumps(row) + "\n" for row in selections))
|
main()
Source code in src/news_recommendation/preparation.py
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172 | def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("stage", choices=("all", "split", "inputs"))
parser.add_argument("--output", required=True, help="Prepared-data directory, relative to recommendation/")
parser.add_argument("--config", default="configs/data_preparation.json")
parser.add_argument("--segmented", help="Segmented article catalog")
parser.add_argument("--section-summaries", help="Segment summaries (outputs.jsonl)")
parser.add_argument("--article-summaries", help="Article-summary run directory")
parser.add_argument("--new-selection", action="store_true",
help="Drop segment summaries at random instead of replaying the saved selection")
args = parser.parse_args()
config = json.loads((REPOSITORY / args.config).read_text())
config |= {key: value for key, value in (("segmented", args.segmented), ("sections", args.section_summaries),
("articles", args.article_summaries)) if value}
if args.new_selection:
del config["selection"]
paths = {key: REPOSITORY / value for key, value in config.items()}
destination = REPOSITORY / args.output
if args.stage in ("all", "split"):
prepare_split(destination, paths)
if args.stage in ("all", "inputs"):
prepare_inputs(destination, paths)
|