Skip to content

parse_guardian

topic_segmentation.news_dataset.parse_guardian

Extract editorial sections from saved Guardian pages.

is_noise_heading(text: str) -> bool

Source code in src/topic_segmentation/news_dataset/parse_guardian.py
68
69
70
def is_noise_heading(text: str) -> bool:
    t = text.lower().strip()
    return len(t) < 3 or any(pat in t for pat in NOISE_PATTERNS)

is_skip_container(el) -> bool

Identify subtrees to exclude from article content.

Source code in src/topic_segmentation/news_dataset/parse_guardian.py
73
74
75
76
77
78
79
80
81
82
def is_skip_container(el) -> bool:
    """Identify subtrees to exclude from article content."""
    # Rich-link figures (related article widgets)
    if el.name == "figure" and el.get("data-spacefinder-role") == "richLink":
        return True
    if el.name == "aside" and el.get("data-gu-name") != "pullquote":
        return True
    if el.name in ("form", "nav"):
        return True
    return False

is_trailing_element(el) -> bool

Source code in src/topic_segmentation/news_dataset/parse_guardian.py
85
86
87
88
89
90
91
92
93
94
95
def is_trailing_element(el) -> bool:
    if el.name in ("h2", "h3", "h4"):
        t = el.get_text(" ", strip=True).lower().rstrip(":")
        if t in TRAILING_HEADINGS:
            return True
        if any(t.startswith(p) for p in TRAILING_HEADING_PREFIXES):
            return True
    text_lower = el.get_text(" ", strip=True).lower()
    if any(text_lower.startswith(p) for p in TRAILING_TEXT_PREFIXES):
        return True
    return False

clean_inner_html(el) -> str

Source code in src/topic_segmentation/news_dataset/parse_guardian.py
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
def clean_inner_html(el) -> str:
    el_copy = copy.deepcopy(el)
    # Replace decorative SVG quote marks with a real quote character
    for svg in el_copy.find_all("svg"):
        svg.replace_with("\u201c")
    for tag in el_copy.find_all(True):
        if tag.name == "script":
            tag.decompose()
            continue
        # Preserve bullet markers
        if tag.name == "span" and tag.get("data-dcr-style") == "bullet":
            tag.attrs = {"data-dcr-style": "bullet"}
            continue
        for attr in STRIP_ATTRS:
            tag.attrs.pop(attr, None)
        if tag.name == "a" and (tag.get("href") or "").startswith("/"):
            tag["href"] = "https://www.theguardian.com" + tag["href"]
            tag["target"] = "_blank"
            tag["rel"] = "noopener"
    return el_copy.decode_contents().strip()

collect_elements(container) -> list

Source code in src/topic_segmentation/news_dataset/parse_guardian.py
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
def collect_elements(container) -> list:
    seen_ids: set = set()
    elements: list = []

    def walk(node):
        for el in node.children:
            if not hasattr(el, "name") or el.name is None:
                continue
            if is_skip_container(el):
                continue
            eid = id(el)
            if eid in seen_ids:
                continue
            seen_ids.add(eid)
            if el.name in BLOCK_TAGS:
                # Recurse into figure wrappers containing headings.
                if el.name == "figure" and el.find(["h2", "h3", "h4"]):
                    walk(el)
                elif not is_link_only_list(el):
                    elements.append(el)
            else:
                walk(el)

    walk(container)
    return elements

process_article(aid: str, raw_html: str, meta: dict)

Source code in src/topic_segmentation/news_dataset/parse_guardian.py
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
def process_article(aid: str, raw_html: str, meta: dict):
    soup = BeautifulSoup(ftfy.fix_text(raw_html), "html.parser")

    body = soup.find(attrs={"data-gu-name": "body"})
    if not body:
        return None

    h1 = soup.find("h1")
    title = h1.get_text(" ", strip=True) if h1 else meta.get("title", "")

    # Standfirst — skip if it's a pure link (newsletter CTA) or boilerplate
    standfirst_text = None
    sf_div = soup.find(attrs={"data-gu-name": "standfirst"})
    if sf_div:
        # Try <p> first, fall back to the whole standfirst div text
        p = sf_div.find("p")
        sf_el = p if p else sf_div
        text = sf_el.get_text(" ", strip=True)
        if text:
            # Skip if entire text is inside a link (e.g. "Sign up now!")
            a_text = "".join(a.get_text(" ", strip=True) for a in sf_el.find_all("a"))
            is_pure_link = a_text and a_text == text
            is_cta = text.lower().startswith("sign up") or "join the conversation" in text.lower()
            if not is_pure_link and not is_cta:
                standfirst_text = text

    # Hero image from data-gu-name="media"
    hero_img_src = None
    media_div = soup.find(attrs={"data-gu-name": "media"})
    if media_div:
        pic = media_div.find("picture")
        if pic:
            img_tag = pic.find("img")
            if img_tag:
                hero_img_src = img_tag.get("src", "")

    sentences, html_parts, labels, title_labels = [], [], [], []

    if standfirst_text:
        sentences.append(standfirst_text)
        html_parts.append({"tag": "p", "html": standfirst_text})
        labels.append(0)
        title_labels.append(1)

    if hero_img_src:
        sentences.append("")
        html_parts.append({"tag": "figure", "html": f'<img src="{hero_img_src}" style="max-width:100%">'})
        labels.append(0)
        title_labels.append(0)

    first_elem = not (standfirst_text or hero_img_src)

    for el in collect_elements(body):
        if is_trailing_element(el):
            break

        # Include figures containing a picture.
        if el.name == "figure":
            pic = el.find("picture")
            if pic:
                img = pic.find("img")
                if img:
                    src = img.get("src", "")
                    if src:
                        figcap = el.find("figcaption")
                        cap_text = ""
                        if figcap:
                            cap_text = figcap.get_text(" ", strip=True)
                        cap_html = f'<figcaption>{cap_text}</figcaption>' if cap_text else ""
                        sentences.append(cap_text)
                        html_parts.append({"tag": "figure", "html": f'<img src="{src}" style="max-width:100%">{cap_html}'})
                        labels.append(0)
                        title_labels.append(0)
                        first_elem = False
            continue

        # Replace decorative SVGs with quote character before extracting text
        for svg in el.find_all("svg"):
            svg.replace_with("\u201c")
        text = el.get_text(" ", strip=True)
        if not text:
            continue

        if el.name in ("h2", "h3", "h4") and is_noise_heading(text):
            continue
        if any(text.lower().startswith(p) for p in SKIP_PARA_PREFIXES):
            continue

        is_heading = el.name in ("h2", "h3", "h4") and len(text) <= 200

        sentences.append(text)
        html_parts.append({"tag": el.name, "html": clean_inner_html(el)})
        if is_heading:
            labels.append(0 if first_elem else 1)
            title_labels.append(1)
        else:
            labels.append(0)
            title_labels.append(0)
        first_elem = False

    # Remove trailing headings without following content.
    while sentences and title_labels[-1] == 1:
        sentences.pop()
        html_parts.pop()
        labels.pop()
        title_labels.pop()

    if not sentences:
        return None

    return {
        "articleId": aid,
        "outlet": "GuardianInt",
        "title": title,
        "url": meta.get("url", ""),
        "sentences": sentences,
        "html_parts": html_parts,
        "labels": labels,
        "title_labels": title_labels,
        "num_sections": labels.count(1) + 1,
    }

main()

Source code in src/topic_segmentation/news_dataset/parse_guardian.py
270
271
272
def main():
    articles = read_articles("guardianint_subheadings", process_article)
    write_articles("guardian_segments.json", unique_articles(articles))