parse_sky_news

topic_segmentation.news_dataset.parse_sky_news

Parse saved pages in data/pages/skynews_subheadings/ into data/outputs/skynews_segments.json.

is_noise_heading(text: str) -> bool

Source code in src/topic_segmentation/news_dataset/parse_sky_news.py

def is_noise_heading(text: str) -> bool:
    t = text.lower().strip()
    return len(t) < 3 or any(pat in t for pat in NOISE_PATTERNS)

is_skip_container(el) -> bool

Source code in src/topic_segmentation/news_dataset/parse_sky_news.py

def is_skip_container(el) -> bool:
    if el.name in ("aside", "nav", "form"):
        return True
    cls = set(el.get("class") or [])
    if cls & SKIP_DIV_CLASSES:
        return True
    return False

is_link_only_p(el) -> bool

Identify paragraphs containing only a link, optionally wrapped in strong.

Source code in src/topic_segmentation/news_dataset/parse_sky_news.py

def is_link_only_p(el) -> bool:
    """Identify paragraphs containing only a link, optionally wrapped in strong."""
    if el.name != "p":
        return False
    direct = "".join(str(c) for c in el.children if c.name is None).strip()
    if direct:
        return False
    children = [c for c in el.children if c.name is not None]
    if len(children) != 1:
        return False
    only = children[0]
    if only.name == "a":
        return True
    if only.name == "strong":
        inner = [c for c in only.children if c.name is not None]
        if len(inner) == 1 and inner[0].name == "a":
            wrap_direct = "".join(str(c) for c in only.children if c.name is None).strip()
            if not wrap_direct:
                return True
    return False

is_link_only_list(el) -> bool

Source code in src/topic_segmentation/news_dataset/parse_sky_news.py

def is_link_only_list(el) -> bool:
    if el.name not in ("ul", "ol"):
        return False
    items = el.find_all("li", recursive=False)
    if not items:
        return False
    for li in items:
        li_text = li.get_text(" ", strip=True)
        a_text = "".join(a.get_text(" ", strip=True) for a in li.find_all("a"))
        if not li_text or not a_text:
            return False
        if li_text == a_text:
            continue
        if len(a_text) / len(li_text) >= 0.6:
            continue
        return False
    return True

is_trailing_element(el) -> bool

Source code in src/topic_segmentation/news_dataset/parse_sky_news.py

def is_trailing_element(el) -> bool:
    text_lower = el.get_text(" ", strip=True).lower()
    if any(text_lower.startswith(p) for p in TRAILING_TEXT_PREFIXES):
        return True
    return False

is_strong_heading(el) -> bool

Recognize a <p><strong> heading of at most 100 characters.

Source code in src/topic_segmentation/news_dataset/parse_sky_news.py

def is_strong_heading(el) -> bool:
    """Recognize a `<p><strong>` heading of at most 100 characters."""
    if el.name != "p":
        return False
    # NavigableString has .name == None; Tags have .name set to the tag name.
    direct_text = "".join(str(c) for c in el.children if c.name is None).strip()
    if direct_text:
        return False
    children = [c for c in el.children if c.name is not None]
    if len(children) != 1 or children[0].name != "strong":
        return False
    txt = el.get_text(" ", strip=True)
    # Short bold leads and Q&A answers serve as headings, including those ending in punctuation.
    if len(txt) > 100:
        return False
    return True

is_heading_noise(txt: str) -> bool

Source code in src/topic_segmentation/news_dataset/parse_sky_news.py

def is_heading_noise(txt: str) -> bool:
    tl = txt.lower()
    if any(tl.startswith(n) for n in HEADING_NOISE_PREFIXES):
        return True
    if tl in HEADING_NOISE_FULL:
        return True
    return False

clean_inner_html(el) -> str

Source code in src/topic_segmentation/news_dataset/parse_sky_news.py

def clean_inner_html(el) -> str:
    el_copy = copy.deepcopy(el)
    for tag in el_copy.find_all(True):
        if tag.name == "script":
            tag.decompose()
            continue
        for attr in STRIP_ATTRS:
            tag.attrs.pop(attr, None)
        if tag.name == "a" and (tag.get("href") or "").startswith("/"):
            tag["href"] = "https://news.sky.com" + tag["href"]
            tag["target"] = "_blank"
            tag["rel"] = "noopener"
    return el_copy.decode_contents().strip()

collect_elements(body) -> list

Collect content elements from sdc-article-body.

Source code in src/topic_segmentation/news_dataset/parse_sky_news.py

def collect_elements(body) -> list:
    """Collect content elements from sdc-article-body."""
    elements = []
    for el in body.children:
        if not hasattr(el, "name") or el.name is None:
            continue
        if is_skip_container(el):
            continue

        cls = set(el.get("class") or [])

        # Inline article image
        if el.name == "div" and "sdc-article-image" in cls:
            fig = el.find("figure")
            if fig:
                img = fig.find("img")
                if img and (img.get("src") or "").startswith("http"):
                    elements.append(fig)
            continue

        # Factbox
        if el.name == "div" and "sdc-article-factbox" in cls:
            elements.append(("factbox", el))
            continue

        # Paragraph or list
        if el.name in ("p", "ul", "ol", "blockquote"):
            if not is_link_only_list(el):
                elements.append(el)
            continue

        # Unknown div/other — skip

    return elements

process_article(aid: str, raw_html: str, meta: dict)

Source code in src/topic_segmentation/news_dataset/parse_sky_news.py

def process_article(aid: str, raw_html: str, meta: dict):
    soup = BeautifulSoup(ftfy.fix_text(raw_html), "html.parser")

    # Title
    h1 = soup.find("h1", class_="sdc-article-header__title")
    if not h1:
        h1 = soup.find("h1")
    title = h1.get_text(" ", strip=True) if h1 else meta.get("title", "")

    # Standfirst
    standfirst_text = None
    sf = soup.find("p", class_="sdc-article-header__sub-title")
    if sf:
        text = sf.get_text(" ", strip=True)
        if text and not text.lower().startswith("sign up"):
            standfirst_text = text

    # Body
    body = soup.find("div", class_="sdc-article-body")
    if not body:
        return None

    # Hero: first sdc-article-image div in body
    hero_img_src = None
    hero_img_cap = None
    first_img_div = body.find("div", class_="sdc-article-image")
    if first_img_div:
        fig = first_img_div.find("figure")
        if fig:
            img = fig.find("img")
            if img and (img.get("src") or "").startswith("http"):
                hero_img_src = img["src"]
                figcap = fig.find("figcaption")
                if figcap:
                    hero_img_cap = figcap.get_text(" ", strip=True)

    sentences, html_parts, labels, title_labels = [], [], [], []

    if standfirst_text:
        sentences.append(standfirst_text)
        html_parts.append({"tag": "p", "html": standfirst_text})
        labels.append(0)
        title_labels.append(1)

    if hero_img_src:
        cap_html = f"<figcaption>{hero_img_cap}</figcaption>" if hero_img_cap else ""
        sentences.append(hero_img_cap or "")
        html_parts.append({"tag": "figure", "html": f'<img src="{hero_img_src}" style="max-width:100%">{cap_html}'})
        labels.append(0)
        title_labels.append(0)

    hero_consumed = False  # skip first sdc-article-image (already used as hero)
    first_elem = not (standfirst_text or hero_img_src)

    # data-highlight-intro marks the first paragraph as an extended standfirst.
    highlight_intro = body.get("data-highlight-intro") == "true"
    intro_p_consumed = not highlight_intro

    for el in collect_elements(body):
        if is_trailing_element(el if not isinstance(el, tuple) else el[1]):
            break

        # Factbox
        if isinstance(el, tuple) and el[0] == "factbox":
            fb = el[1]
            fb_title = fb.find("h4")
            fb_text = fb.get_text(" ", strip=True)
            if not fb_text:
                continue
            title_html = f"<strong>{fb_title.get_text(' ', strip=True)}</strong><br>" if fb_title else ""
            sentences.append(fb_text)
            html_parts.append({"tag": "blockquote", "html": title_html + fb_text})
            labels.append(0)
            title_labels.append(0)
            first_elem = False
            continue

        # Figure (body image) — skip hero on first encounter
        if el.name == "figure":
            img = el.find("img")
            if not img:
                continue
            src = img.get("src", "")
            if not src:
                continue
            if not hero_consumed and hero_img_src and src == hero_img_src:
                hero_consumed = True
                continue
            figcap = el.find("figcaption")
            cap_text = figcap.get_text(" ", strip=True) if figcap else ""
            cap_html = f"<figcaption>{cap_text}</figcaption>" if cap_text else ""
            sentences.append(cap_text)
            html_parts.append({"tag": "figure", "html": f'<img src="{src}" style="max-width:100%">{cap_html}'})
            labels.append(0)
            title_labels.append(0)
            first_elem = False
            continue

        # Remove link-only promotional paragraphs.
        if is_link_only_p(el):
            continue

        # Strong heading paragraph
        if is_strong_heading(el):
            text = el.get_text(" ", strip=True)
            if not text or is_noise_heading(text) or is_heading_noise(text):
                continue
            sentences.append(text)
            html_parts.append({"tag": "h3", "html": text})
            labels.append(0 if first_elem else 1)
            title_labels.append(1)
            first_elem = False
            continue

        text = el.get_text(" ", strip=True)
        if not text:
            continue
        if any(text.lower().startswith(p) for p in SKIP_PARA_PREFIXES):
            continue

        # First body <p> in a data-highlight-intro="true" article = styled lede
        is_intro = el.name == "p" and not intro_p_consumed
        if is_intro:
            intro_p_consumed = True

        sentences.append(text)
        html_parts.append({"tag": el.name, "html": clean_inner_html(el)})
        labels.append(0)
        title_labels.append(1 if is_intro else 0)
        first_elem = False

    # Remove trailing headings without following content.
    while sentences and title_labels[-1] == 1:
        sentences.pop()
        html_parts.pop()
        labels.pop()
        title_labels.pop()

    if not sentences:
        return None

    return {
        "articleId": aid,
        "outlet": "SkyNews",
        "title": title,
        "url": meta.get("url", ""),
        "sentences": sentences,
        "html_parts": html_parts,
        "labels": labels,
        "title_labels": title_labels,
        "num_sections": labels.count(1) + 1,
    }

main()

Source code in src/topic_segmentation/news_dataset/parse_sky_news.py

def main():
    with open(ROOT / "data/articles.json", encoding="latin-1") as f:
        all_meta = {a["articleId"]: a for a in json.load(f)}

    results = []
    seen_content: set = set()
    skipped = 0
    for html_file in sorted((ROOT / "data/pages/skynews_subheadings").glob("*.html")):
        aid = html_file.stem
        raw = html_file.read_text(encoding="utf-8")
        rec = process_article(aid, raw, all_meta.get(aid, {}))
        if not rec:
            skipped += 1
            continue
        content = (rec.get("title", "") + "".join(rec["sentences"]))
        if content in seen_content:
            skipped += 1
            continue
        seen_content.add(content)
        results.append(rec)

    one_section = sum(1 for r in results if r["num_sections"] == 1)
    print(f"Processed {len(results)} articles ({skipped} skipped/duplicate)")
    print(f"  1 section: {one_section}, 2+ sections: {len(results) - one_section}")
    results = [r for r in results if r["num_sections"] >= 2]

    (ROOT / "data/outputs").mkdir(parents=True, exist_ok=True)
    seg_path = ROOT / "data/outputs/skynews_segments.json"
    with open(seg_path, "w", encoding="utf-8") as f:
        json.dump(results, f, ensure_ascii=False, indent=2)
    print(f"Saved {seg_path}")