173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319 | def process_article(aid: str, raw_html: str, meta: dict):
soup = BeautifulSoup(ftfy.fix_text(raw_html), "html.parser")
h1 = soup.find("h1", class_="sdc-article-header__title")
if not h1:
h1 = soup.find("h1")
title = h1.get_text(" ", strip=True) if h1 else meta.get("title", "")
standfirst_text = None
sf = soup.find("p", class_="sdc-article-header__sub-title")
if sf:
text = sf.get_text(" ", strip=True)
if text and not text.lower().startswith("sign up"):
standfirst_text = text
body = soup.find("div", class_="sdc-article-body")
if not body:
return None
# Hero: first sdc-article-image div in body
hero_img_src = None
hero_img_cap = None
first_img_div = body.find("div", class_="sdc-article-image")
if first_img_div:
fig = first_img_div.find("figure")
if fig:
img = fig.find("img")
if img and (img.get("src") or "").startswith("http"):
hero_img_src = img["src"]
figcap = fig.find("figcaption")
if figcap:
hero_img_cap = figcap.get_text(" ", strip=True)
sentences, html_parts, labels, title_labels = [], [], [], []
if standfirst_text:
sentences.append(standfirst_text)
html_parts.append({"tag": "p", "html": standfirst_text})
labels.append(0)
title_labels.append(1)
if hero_img_src:
cap_html = f"<figcaption>{hero_img_cap}</figcaption>" if hero_img_cap else ""
sentences.append(hero_img_cap or "")
html_parts.append({"tag": "figure", "html": f'<img src="{hero_img_src}" style="max-width:100%">{cap_html}'})
labels.append(0)
title_labels.append(0)
hero_consumed = False # skip first sdc-article-image (already used as hero)
first_elem = not (standfirst_text or hero_img_src)
# data-highlight-intro marks the first paragraph as an extended standfirst.
highlight_intro = body.get("data-highlight-intro") == "true"
intro_p_consumed = not highlight_intro
for el in collect_elements(body):
if is_trailing_element(el if not isinstance(el, tuple) else el[1]):
break
if isinstance(el, tuple) and el[0] == "factbox":
fb = el[1]
fb_title = fb.find("h4")
fb_text = fb.get_text(" ", strip=True)
if not fb_text:
continue
title_html = f"<strong>{fb_title.get_text(' ', strip=True)}</strong><br>" if fb_title else ""
sentences.append(fb_text)
html_parts.append({"tag": "blockquote", "html": title_html + fb_text})
labels.append(0)
title_labels.append(0)
first_elem = False
continue
# Figure (body image) — skip hero on first encounter
if el.name == "figure":
img = el.find("img")
if not img:
continue
src = img.get("src", "")
if not src:
continue
if not hero_consumed and hero_img_src and src == hero_img_src:
hero_consumed = True
continue
figcap = el.find("figcaption")
cap_text = figcap.get_text(" ", strip=True) if figcap else ""
cap_html = f"<figcaption>{cap_text}</figcaption>" if cap_text else ""
sentences.append(cap_text)
html_parts.append({"tag": "figure", "html": f'<img src="{src}" style="max-width:100%">{cap_html}'})
labels.append(0)
title_labels.append(0)
first_elem = False
continue
# Remove link-only promotional paragraphs.
if is_link_only_p(el):
continue
if is_strong_heading(el):
text = el.get_text(" ", strip=True)
if not text or is_noise_heading(text) or is_heading_noise(text):
continue
sentences.append(text)
html_parts.append({"tag": "h3", "html": text})
labels.append(0 if first_elem else 1)
title_labels.append(1)
first_elem = False
continue
text = el.get_text(" ", strip=True)
if not text:
continue
if any(text.lower().startswith(p) for p in SKIP_PARA_PREFIXES):
continue
# First body <p> in a data-highlight-intro="true" article = styled lede
is_intro = el.name == "p" and not intro_p_consumed
if is_intro:
intro_p_consumed = True
sentences.append(text)
html_parts.append({"tag": el.name, "html": clean_inner_html(el, "https://news.sky.com")})
labels.append(0)
title_labels.append(1 if is_intro else 0)
first_elem = False
# Remove trailing headings without following content.
while sentences and title_labels[-1] == 1:
sentences.pop()
html_parts.pop()
labels.pop()
title_labels.pop()
if not sentences:
return None
return {
"articleId": aid,
"outlet": "SkyNews",
"title": title,
"url": meta.get("url", ""),
"sentences": sentences,
"html_parts": html_parts,
"labels": labels,
"title_labels": title_labels,
"num_sections": labels.count(1) + 1,
}
|