147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267 | def process_article(aid: str, raw_html: str, meta: dict):
soup = BeautifulSoup(ftfy.fix_text(raw_html), "html.parser")
body = soup.find(attrs={"data-gu-name": "body"})
if not body:
return None
h1 = soup.find("h1")
title = h1.get_text(" ", strip=True) if h1 else meta.get("title", "")
# Standfirst — skip if it's a pure link (newsletter CTA) or boilerplate
standfirst_text = None
sf_div = soup.find(attrs={"data-gu-name": "standfirst"})
if sf_div:
# Try <p> first, fall back to the whole standfirst div text
p = sf_div.find("p")
sf_el = p if p else sf_div
text = sf_el.get_text(" ", strip=True)
if text:
# Skip if entire text is inside a link (e.g. "Sign up now!")
a_text = "".join(a.get_text(" ", strip=True) for a in sf_el.find_all("a"))
is_pure_link = a_text and a_text == text
is_cta = text.lower().startswith("sign up") or "join the conversation" in text.lower()
if not is_pure_link and not is_cta:
standfirst_text = text
# Hero image from data-gu-name="media"
hero_img_src = None
media_div = soup.find(attrs={"data-gu-name": "media"})
if media_div:
pic = media_div.find("picture")
if pic:
img_tag = pic.find("img")
if img_tag:
hero_img_src = img_tag.get("src", "")
sentences, html_parts, labels, title_labels = [], [], [], []
if standfirst_text:
sentences.append(standfirst_text)
html_parts.append({"tag": "p", "html": standfirst_text})
labels.append(0)
title_labels.append(1)
if hero_img_src:
sentences.append("")
html_parts.append({"tag": "figure", "html": f'<img src="{hero_img_src}" style="max-width:100%">'})
labels.append(0)
title_labels.append(0)
first_elem = not (standfirst_text or hero_img_src)
for el in collect_elements(body):
if is_trailing_element(el):
break
# Include figures containing a picture.
if el.name == "figure":
pic = el.find("picture")
if pic:
img = pic.find("img")
if img:
src = img.get("src", "")
if src:
figcap = el.find("figcaption")
cap_text = ""
if figcap:
cap_text = figcap.get_text(" ", strip=True)
cap_html = f'<figcaption>{cap_text}</figcaption>' if cap_text else ""
sentences.append(cap_text)
html_parts.append({"tag": "figure", "html": f'<img src="{src}" style="max-width:100%">{cap_html}'})
labels.append(0)
title_labels.append(0)
first_elem = False
continue
# Replace decorative SVGs with quote character before extracting text
for svg in el.find_all("svg"):
svg.replace_with("\u201c")
text = el.get_text(" ", strip=True)
if not text:
continue
if el.name in ("h2", "h3", "h4") and is_noise_heading(text):
continue
if any(text.lower().startswith(p) for p in SKIP_PARA_PREFIXES):
continue
is_heading = el.name in ("h2", "h3", "h4") and len(text) <= 200
sentences.append(text)
html_parts.append({"tag": el.name, "html": clean_inner_html(el)})
if is_heading:
labels.append(0 if first_elem else 1)
title_labels.append(1)
else:
labels.append(0)
title_labels.append(0)
first_elem = False
# Remove trailing headings without following content.
while sentences and title_labels[-1] == 1:
sentences.pop()
html_parts.pop()
labels.pop()
title_labels.pop()
if not sentences:
return None
return {
"articleId": aid,
"outlet": "GuardianInt",
"title": title,
"url": meta.get("url", ""),
"sentences": sentences,
"html_parts": html_parts,
"labels": labels,
"title_labels": title_labels,
"num_sections": labels.count(1) + 1,
}
|