parse_bbc

topic_segmentation.news_dataset.parse_bbc

Parse saved pages in data/pages/bbc_subheadings/ into data/outputs/bbc_segments.json.

is_link_only_list(el) -> bool

Identify lists whose items each have at least 60% linked text.

Source code in src/topic_segmentation/news_dataset/parse_bbc.py

def is_link_only_list(el) -> bool:
    """Identify lists whose items each have at least 60% linked text."""
    if el.name not in ("ul", "ol"):
        return False
    items = el.find_all("li", recursive=False)
    if not items:
        return False
    for li in items:
        li_text = li.get_text(" ", strip=True)
        a_text = "".join(a.get_text(" ", strip=True) for a in li.find_all("a"))
        if not li_text or not a_text:
            return False
        if li_text == a_text:
            continue  # pure link
        if len(a_text) / len(li_text) >= 0.6:
            continue  # short label prefix + link
        return False
    return True

is_noise_heading(text: str) -> bool

Source code in src/topic_segmentation/news_dataset/parse_bbc.py

def is_noise_heading(text: str) -> bool:
    t = text.lower().strip()
    return len(t) < 3 or any(pat in t for pat in NOISE_PATTERNS)

is_skip_container(el) -> bool

Identify subtrees to exclude from article content.

Source code in src/topic_segmentation/news_dataset/parse_bbc.py

def is_skip_container(el) -> bool:
    """Identify subtrees to exclude from article content."""
    el_id = (el.get("id") or "").lower()
    if any(s in el_id for s in SKIP_ID_SUBSTRINGS):
        return True
    if el.get("data-testid") in SKIP_TEST_IDS:
        return True
    if el.get("data-component") in SKIP_COMPONENTS:
        return True
    if el.get("data-block") in SKIP_BLOCKS:
        return True
    # Skip video player figures (contain "To play this video..." boilerplate)
    if el.name == "figure" and el.find(
            attrs={"data-testid": "media-player-container-landscape"}):
        return True
    return False

is_trailing_element(el) -> bool

Identify the start of trailing boilerplate.

Source code in src/topic_segmentation/news_dataset/parse_bbc.py

def is_trailing_element(el) -> bool:
    """Identify the start of trailing boilerplate."""
    # Match trailing headings by text or prefix.
    if el.name in ("h2", "h3", "h4"):
        text_h = el.get_text(" ", strip=True).lower().rstrip(":")
        if text_h in TRAILING_HEADINGS:
            return True
        if any(text_h.startswith(p) for p in TRAILING_HEADING_PREFIXES):
            return True

    # Check the element and its descendants for boilerplate markers.
    for tag in [el, *el.find_all(True)]:
        if any(s in (tag.get("id") or "").lower() for s in SKIP_ID_SUBSTRINGS):
            return True
        if tag.get("data-testid") in SKIP_TEST_IDS:
            return True
        if any(pat in (tag.get("alt") or "").lower()
               for pat in TRAILING_TEXT_PATTERNS):
            return True

    # Treat italicized paragraphs as follow prompts only when they contain links.
    if el.name == "p":
        children = [c for c in el.children if hasattr(c, "name") and c.name]
        has_link = any(c.name == "a" for c in children)
        if has_link and children[0].name in ("em", "i") and all(
                c.name in ("em", "i", "a") for c in children):
            return True

    text_lower = el.get_text(" ", strip=True).lower()
    if any(pat in text_lower for pat in TRAILING_TEXT_PATTERNS):
        return True
    if any(text_lower.startswith(pat) for pat in TRAILING_TEXT_PREFIXES):
        return True

    return False

clean_inner_html(el) -> str

Remove unwanted HTML attributes and resolve relative BBC links.

Source code in src/topic_segmentation/news_dataset/parse_bbc.py

def clean_inner_html(el) -> str:
    """Remove unwanted HTML attributes and resolve relative BBC links."""
    el_copy = copy.deepcopy(el)
    for tag in el_copy.find_all(True):
        if tag.name == "script":
            tag.decompose()
            continue
        for attr in STRIP_ATTRS:
            tag.attrs.pop(attr, None)
        if tag.name == "a" and (tag.get("href") or "").startswith("/"):
            tag["href"] = "https://www.bbc.co.uk" + tag["href"]
            tag["target"] = "_blank"
            tag["rel"] = "noopener"
    return el_copy.decode_contents().strip()

collect_elements(container) -> list

Collect content in document order, skipping excluded subtrees and stopping at trailing boilerplate.

Source code in src/topic_segmentation/news_dataset/parse_bbc.py

def collect_elements(container) -> list:
    """Collect content in document order, skipping excluded subtrees and stopping at trailing boilerplate."""
    seen_ids: set = set()
    elements: list = []

    def walk(node):
        for el in node.children:
            if not hasattr(el, "name") or el.name is None:
                continue
            if is_skip_container(el):
                continue
            eid = id(el)
            if eid in seen_ids:
                continue
            seen_ids.add(eid)
            if el.name in BLOCK_TAGS:
                if not is_link_only_list(el):
                    elements.append(el)
            else:
                walk(el)

    walk(container)
    return elements

process_article(aid: str, raw_html: str, meta: dict)

Source code in src/topic_segmentation/news_dataset/parse_bbc.py

def process_article(aid: str, raw_html: str, meta: dict):
    soup = BeautifulSoup(ftfy.fix_text(raw_html), "html.parser")

    article = soup.select_one("article")
    if not article:
        return None
    first_div = article.find("div", recursive=False)
    container = first_div if first_div else article

    h1 = soup.select_one("h1")
    title = h1.get_text(" ", strip=True) if h1 else meta.get("title", "")

    sentences, html_parts, labels, title_labels = [], [], [], []
    first_elem = True

    for el in collect_elements(container):
        if is_trailing_element(el):
            break
        # For figures: strip "Image source,X" attribution, keep only the caption
        if el.name == "figure":
            figcap = el.find("figcaption")
            if not figcap:
                continue
            # Remove attribution spans (visually-hidden "Image source,..." nodes)
            import copy as _copy
            cap = _copy.deepcopy(figcap)
            for span in cap.find_all(True):
                if "image source" in span.get_text(" ", strip=True).lower()[:20]:
                    span.decompose()
            text = cap.get_text(" ", strip=True)
        else:
            text = el.get_text(" ", strip=True)
        if not text:
            continue

        # Skip promotional headings and continue collecting content.
        if el.name in ("h2", "h3", "h4") and is_noise_heading(text):
            continue
        # Link-only boilerplate paragraphs (e.g. "Read more here")
        if el.name == "p" and text.lower().rstrip(".") in SKIP_PARA_TEXT:
            continue
        # Mid-article promo paragraphs (e.g. "The documentary is available…")
        if el.name == "p" and any(text.lower().startswith(p)
                                  for p in SKIP_PARA_PREFIXES):
            continue
        # BBC series and brand promos
        if any(text.lower().startswith(p) for p in SKIP_ANY_PREFIXES):
            continue

        is_heading = (el.name in ("h2", "h3", "h4") and len(text) <= 200)
        sentences.append(text)
        html_parts.append({"tag": el.name, "html": clean_inner_html(el)})
        if is_heading:
            labels.append(0 if first_elem else 1)
            title_labels.append(1)
        else:
            labels.append(0)
            title_labels.append(0)
        first_elem = False

    # Remove trailing headings without following content.
    while sentences and title_labels[-1] == 1:
        sentences.pop()
        html_parts.pop()
        labels.pop()
        title_labels.pop()

    if not sentences:
        return None

    return {
        "articleId": aid,
        "outlet": "BBC",
        "title": title,
        "url": meta.get("url", ""),
        "sentences": sentences,
        "html_parts": html_parts,
        "labels": labels,
        "title_labels": title_labels,
        "num_sections": labels.count(1) + 1,
    }

main()

Source code in src/topic_segmentation/news_dataset/parse_bbc.py

def main():
    with open(ROOT / "data/articles.json", encoding="latin-1") as f:
        all_meta = {a["articleId"]: a for a in json.load(f)}

    results = []
    for html_file in sorted((ROOT / "data/pages/bbc_subheadings").glob("*.html")):
        aid = html_file.stem
        raw = html_file.read_text(encoding="utf-8")
        rec = process_article(aid, raw, all_meta.get(aid, {}))
        if rec:
            results.append(rec)

    results = [r for r in results if r["num_sections"] >= 2]
    print(f"Processed {len(results)} articles")

    (ROOT / "data/outputs").mkdir(parents=True, exist_ok=True)
    seg_path = ROOT / "data/outputs/bbc_segments.json"
    with open(seg_path, "w", encoding="utf-8") as f:
        json.dump(results, f, ensure_ascii=False, indent=2)
    print(f"Saved {seg_path}")