Skip to content

model

news_recommendation.model

PLM-NR (Wu et al., 2021) with a BERT news encoder, adapted from https://github.com/wuch15/PLM4NewsRec.

Only the title encoder is kept: each input variant packs its text into the title field. Attribute names match the state-dict keys of the trained checkpoints, and modules are built in the upstream order, so a seeded run draws the same initial weights.

AdditiveAttention

Bases: Module

Attention pooling over the second dimension.

Source code in src/news_recommendation/model.py
13
14
15
16
17
18
19
20
21
22
23
24
class AdditiveAttention(nn.Module):
    """Attention pooling over the second dimension."""

    def __init__(self, dimension, hidden_size=200):
        super().__init__()
        self.att_fc1 = nn.Linear(dimension, hidden_size)
        self.att_fc2 = nn.Linear(hidden_size, 1)

    def forward(self, x):
        alpha = torch.exp(self.att_fc2(torch.tanh(self.att_fc1(x))))
        alpha = alpha / (torch.sum(alpha, dim=1, keepdim=True) + 1e-8)
        return torch.bmm(x.permute(0, 2, 1), alpha).reshape(x.shape[0], -1)

att_fc1 = nn.Linear(dimension, hidden_size) instance-attribute

att_fc2 = nn.Linear(hidden_size, 1) instance-attribute

forward(x)

Source code in src/news_recommendation/model.py
21
22
23
24
def forward(self, x):
    alpha = torch.exp(self.att_fc2(torch.tanh(self.att_fc1(x))))
    alpha = alpha / (torch.sum(alpha, dim=1, keepdim=True) + 1e-8)
    return torch.bmm(x.permute(0, 2, 1), alpha).reshape(x.shape[0], -1)

MultiHeadAttention

Bases: Module

Unmasked multi-head self-attention with exp-normalized scores, as upstream.

Source code in src/news_recommendation/model.py
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
class MultiHeadAttention(nn.Module):
    """Unmasked multi-head self-attention with exp-normalized scores, as upstream."""

    def __init__(self, dimension, heads, head_size):
        super().__init__()
        self.heads, self.head_size = heads, head_size
        self.W_Q = nn.Linear(dimension, head_size * heads)
        self.W_K = nn.Linear(dimension, head_size * heads)
        self.W_V = nn.Linear(dimension, head_size * heads)
        for linear in (self.W_Q, self.W_K, self.W_V):
            nn.init.xavier_uniform_(linear.weight, gain=1)

    def forward(self, x):
        batch_size = x.shape[0]
        q, k, v = (linear(x).view(batch_size, -1, self.heads, self.head_size).transpose(1, 2)
                   for linear in (self.W_Q, self.W_K, self.W_V))
        scores = torch.exp(torch.matmul(q, k.transpose(-1, -2)) / np.sqrt(self.head_size))
        attention = scores / (torch.sum(scores, dim=-1, keepdim=True) + 1e-8)
        return torch.matmul(attention, v).transpose(1, 2).contiguous().view(batch_size, -1, self.heads * self.head_size)

W_Q = nn.Linear(dimension, head_size * heads) instance-attribute

W_K = nn.Linear(dimension, head_size * heads) instance-attribute

W_V = nn.Linear(dimension, head_size * heads) instance-attribute

forward(x)

Source code in src/news_recommendation/model.py
39
40
41
42
43
44
45
def forward(self, x):
    batch_size = x.shape[0]
    q, k, v = (linear(x).view(batch_size, -1, self.heads, self.head_size).transpose(1, 2)
               for linear in (self.W_Q, self.W_K, self.W_V))
    scores = torch.exp(torch.matmul(q, k.transpose(-1, -2)) / np.sqrt(self.head_size))
    attention = scores / (torch.sum(scores, dim=-1, keepdim=True) + 1e-8)
    return torch.matmul(attention, v).transpose(1, 2).contiguous().view(batch_size, -1, self.heads * self.head_size)

TextEncoder

Bases: Module

BERT token states, multi-head self-attention, and attention pooling of one packed text.

Source code in src/news_recommendation/model.py
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
class TextEncoder(nn.Module):
    """BERT token states, multi-head self-attention, and attention pooling of one packed text."""

    def __init__(self, bert_model, config):
        super().__init__()
        self.bert_model = bert_model
        self.dropout_rate = config["drop_rate"]
        self.multihead_attention = MultiHeadAttention(config["word_embedding_dim"], config["num_attention_heads"], 20)
        self.additive_attention = AdditiveAttention(config["num_attention_heads"] * 20, config["news_query_vector_dim"])

    def forward(self, text):
        """`text` packs input ids, token type ids, and attention mask side by side."""
        ids, types, mask = text.chunk(3, dim=1)
        words = self.bert_model(input_ids=ids, token_type_ids=types, attention_mask=mask, return_dict=True).last_hidden_state
        words = F.dropout(words, p=self.dropout_rate, training=self.training)
        words = F.dropout(self.multihead_attention(words), p=self.dropout_rate, training=self.training)
        return self.additive_attention(words)

bert_model = bert_model instance-attribute

dropout_rate = config['drop_rate'] instance-attribute

multihead_attention = MultiHeadAttention(config['word_embedding_dim'], config['num_attention_heads'], 20) instance-attribute

additive_attention = AdditiveAttention(config['num_attention_heads'] * 20, config['news_query_vector_dim']) instance-attribute

forward(text)

text packs input ids, token type ids, and attention mask side by side.

Source code in src/news_recommendation/model.py
58
59
60
61
62
63
64
def forward(self, text):
    """`text` packs input ids, token type ids, and attention mask side by side."""
    ids, types, mask = text.chunk(3, dim=1)
    words = self.bert_model(input_ids=ids, token_type_ids=types, attention_mask=mask, return_dict=True).last_hidden_state
    words = F.dropout(words, p=self.dropout_rate, training=self.training)
    words = F.dropout(self.multihead_attention(words), p=self.dropout_rate, training=self.training)
    return self.additive_attention(words)

NewsEncoder

Bases: Module

Source code in src/news_recommendation/model.py
67
68
69
70
71
72
73
74
class NewsEncoder(nn.Module):
    def __init__(self, bert_model, config):
        super().__init__()
        self.text_encoders = nn.ModuleDict({"title": TextEncoder(bert_model, config)})
        self.reduce_dim_linear = nn.Linear(config["num_attention_heads"] * 20, config["news_dim"])

    def forward(self, news):
        return self.reduce_dim_linear(self.text_encoders["title"](news))

text_encoders = nn.ModuleDict({'title': TextEncoder(bert_model, config)}) instance-attribute

reduce_dim_linear = nn.Linear(config['num_attention_heads'] * 20, config['news_dim']) instance-attribute

forward(news)

Source code in src/news_recommendation/model.py
73
74
def forward(self, news):
    return self.reduce_dim_linear(self.text_encoders["title"](news))

UserEncoder

Bases: Module

Attention pooling over all history slots; padded slots hold the encoded all-zero news row.

Source code in src/news_recommendation/model.py
77
78
79
80
81
82
83
84
85
class UserEncoder(nn.Module):
    """Attention pooling over all history slots; padded slots hold the encoded all-zero news row."""

    def __init__(self, config):
        super().__init__()
        self.news_additive_attention = AdditiveAttention(config["news_dim"], config["user_query_vector_dim"])

    def forward(self, history):
        return self.news_additive_attention(history)

news_additive_attention = AdditiveAttention(config['news_dim'], config['user_query_vector_dim']) instance-attribute

forward(history)

Source code in src/news_recommendation/model.py
84
85
def forward(self, history):
    return self.news_additive_attention(history)

ModelBert

Bases: Module

Click-prediction loss over 1 + K candidates and the user's history.

Source code in src/news_recommendation/model.py
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
class ModelBert(nn.Module):
    """Click-prediction loss over 1 + K candidates and the user's history."""

    def __init__(self, config, bert_model):
        super().__init__()
        self.npratio, self.user_log_length, self.news_dim = config["npratio"], config["user_log_length"], config["news_dim"]
        self.news_encoder = NewsEncoder(bert_model, config)
        self.user_encoder = UserEncoder(config)
        self.criterion = nn.CrossEntropyLoss()

    def forward(self, candidates, history, history_mask, targets):
        length = candidates.size(2)
        news = self.news_encoder(candidates.view(-1, length)).view(-1, 1 + self.npratio, self.news_dim)
        # Padded history slots hold the all-zero news row; encode it once and reuse it.
        history = history.view(-1, length)
        real = history_mask.reshape(-1) > 0
        vectors = self.news_encoder(history[real])
        if not real.all():
            padding = self.news_encoder(torch.zeros_like(history[:1]))
            vectors = padding.expand(len(real), -1).index_put((real.nonzero().squeeze(1),), vectors)
        user = self.user_encoder(vectors.view(-1, self.user_log_length, self.news_dim))
        score = torch.bmm(news, user.unsqueeze(-1)).squeeze(dim=-1)
        return self.criterion(score, targets)

news_encoder = NewsEncoder(bert_model, config) instance-attribute

user_encoder = UserEncoder(config) instance-attribute

criterion = nn.CrossEntropyLoss() instance-attribute

forward(candidates, history, history_mask, targets)

Source code in src/news_recommendation/model.py
 98
 99
100
101
102
103
104
105
106
107
108
109
110
def forward(self, candidates, history, history_mask, targets):
    length = candidates.size(2)
    news = self.news_encoder(candidates.view(-1, length)).view(-1, 1 + self.npratio, self.news_dim)
    # Padded history slots hold the all-zero news row; encode it once and reuse it.
    history = history.view(-1, length)
    real = history_mask.reshape(-1) > 0
    vectors = self.news_encoder(history[real])
    if not real.all():
        padding = self.news_encoder(torch.zeros_like(history[:1]))
        vectors = padding.expand(len(real), -1).index_put((real.nonzero().squeeze(1),), vectors)
    user = self.user_encoder(vectors.view(-1, self.user_log_length, self.news_dim))
    score = torch.bmm(news, user.unsqueeze(-1)).squeeze(dim=-1)
    return self.criterion(score, targets)