Merge parsed articles from data/outputs/*_segments.json into data/dataset.jsonl.
Add block_types (standfirst, h2, h3, image, element) and image subtypes (hero, body).
Map h4 and other heading tags to h3. Store the headline separately in title.
Normalize list formatting and remove duplicate content across outlets.
python -m topic_segmentation.news_dataset.gen_dataset
normalise(text: str) -> str
Normalize whitespace and punctuation spacing for content comparison.
Source code in src/topic_segmentation/news_dataset/gen_dataset.py
| def normalise(text: str) -> str:
"""Normalize whitespace and punctuation spacing for content comparison."""
return SPACE_BEFORE_PUNCT_RE.sub(r"\1", WHITESPACE_RE.sub(" ", text)).strip()
|
reformat_list_body(html: str, ordered: bool) -> str
Convert a <ul>/<ol> HTML fragment into bullet- or number-prefixed lines.
Source code in src/topic_segmentation/news_dataset/gen_dataset.py
40
41
42
43
44
45
46
47
48
49 | def reformat_list_body(html: str, ordered: bool) -> str:
"""Convert a `<ul>`/`<ol>` HTML fragment into bullet- or number-prefixed lines."""
soup = BeautifulSoup(html, "html.parser")
items = [li.get_text(" ", strip=True) for li in soup.find_all("li")]
items = [it for it in items if it]
if not items:
return ""
if ordered:
return "\n".join(f"{i + 1}. {it}" for i, it in enumerate(items))
return "\n".join(f"- {it}" for it in items)
|
classify_record(record: dict) -> tuple[list, list]
Source code in src/topic_segmentation/news_dataset/gen_dataset.py
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88 | def classify_record(record: dict) -> tuple[list, list]:
block_types = []
block_subtypes = []
seen_figure = False
parts = record.get("html_parts", [])
title_labels = record.get("title_labels", [])
for i, part in enumerate(parts):
tag = part.get("tag", "")
html = part.get("html", "")
is_title = bool(title_labels[i]) if i < len(title_labels) else False
# BBC's initial bold paragraph is a standfirst with title_labels=0.
is_bbc_bold_lead = (
i == 0 and tag == "p" and html.lstrip().startswith("<b>")
)
sub = None
if tag == "figure":
t = "image"
sub = "hero" if not seen_figure else "body"
seen_figure = True
elif (is_title and tag == "p") or is_bbc_bold_lead:
t = "standfirst"
elif is_title and tag == "h2":
t = "h2"
elif is_title and tag in ("h3", "h4"):
t = "h3"
elif is_title:
t = "h3" # Other heading tags map to h3.
else:
t = "element"
block_types.append(t)
block_subtypes.append(sub)
return block_types, block_subtypes
|
main()
Source code in src/topic_segmentation/news_dataset/gen_dataset.py
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141 | def main():
counts: dict = {}
total = 0
extra_dupes = 0
seen_content: set = set()
with DATASET_PATH.open("w", encoding="utf-8") as out:
for fname in SEGMENTS_FILES:
p = OUTPUTS_DIR / fname
if not p.exists():
print(f" [skip] {fname} not found")
continue
records = json.loads(p.read_text())
kept = 0
for r in records:
r["block_types"], r["block_subtypes"] = classify_record(r)
# Use body for the article text.
if "sentences" in r:
r["body"] = r.pop("sentences")
# Reformat list bodies as "- item" / "1. item" lines.
for i, part in enumerate(r["html_parts"]):
if part.get("tag") in ("ul", "ol"):
r["body"][i] = reformat_list_body(
part.get("html", ""), ordered=part["tag"] == "ol"
)
# Deduplicate across outlets after normalizing whitespace and punctuation spacing.
content = normalise(r.get("title", "") + "".join(r["body"]))
if content in seen_content:
extra_dupes += 1
continue
seen_content.add(content)
for t in r["block_types"]:
counts[t] = counts.get(t, 0) + 1
for s in r["block_subtypes"]:
if s:
counts[f"image:{s}"] = counts.get(f"image:{s}", 0) + 1
out.write(json.dumps(r, ensure_ascii=False) + "\n")
total += 1
kept += 1
print(f" {fname}: {len(records)} → {kept} kept")
if extra_dupes:
print(f" ({extra_dupes} duplicate(s) caught by normalised content comparison)")
size_mb = os.path.getsize(DATASET_PATH) / 1024 / 1024
print()
print(f"Saved {DATASET_PATH} ({size_mb:.1f} MB) — {total} articles total")
print()
print("Block-type totals:")
for k in ["standfirst", "h2", "h3", "image", "image:hero", "image:body", "element"]:
if k in counts:
print(f" {k}: {counts[k]:,}")
|