📖 第10课：GloVe

import numpy as np from collections import defaultdict class SimpleGloVe: def __init__(self, vector_size=20, alpha=0.75, x_max=10): self.vector_size = vector_size self.alpha = alpha self.x_max = x_max def build_cooccurrence(self, corpus, window=2): cooccur = defaultdict(float) for tokens in corpus: for i, center in enumerate(tokens): for j in range(max(0, i-window), min(len(tokens), i+window+1)): if i != j: weight = 1.0 / abs(i - j) cooccur[(center, tokens[j])] += weight return cooccur def weighting_func(self, x): if x < self.x_max: return (x / self.x_max) ** self.alpha return 1.0 def fit(self, corpus, window=2, epochs=100, lr=0.01): cooccur = self.build_cooccurrence(corpus, window) vocab = sorted(set(w for pair in cooccur for w in pair)) word2idx = {w: i for i, w in enumerate(vocab)} n = len(vocab) W = np.random.randn(n, self.vector_size) * 0.1 W_ctx = np.random.randn(n, self.vector_size) * 0.1 b = np.zeros(n) b_ctx = np.zeros(n) pairs = [(word2idx[w1], word2idx[w2], count) for (w1, w2), count in cooccur.items()] for epoch in range(epochs): total_loss = 0 for i, j, x_ij in pairs: diff = np.dot(W[i], W_ctx[j]) + b[i] + b_ctx[j] - np.log(x_ij) f_x = self.weighting_func(x_ij) total_loss += f_x * diff ** 2 grad_main = f_x * diff * W_ctx[j] grad_ctx = f_x * diff * W[i] W[i] -= lr * grad_main W_ctx[j] -= lr * grad_ctx b[i] -= lr * f_x * diff b_ctx[j] -= lr * f_x * diff if (epoch + 1) % 20 == 0: print(f"Epoch {epoch+1}: loss = {total_loss:.4f}") self.W = W + W_ctx self.word2idx = word2idx self.vocab = vocab return self def most_similar(self, word, topn=5): if word not in self.word2idx: return [] vec = self.W[self.word2idx[word]] norms = np.linalg.norm(self.W, axis=1, keepdims=True) norms = np.where(norms == 0, 1, norms) normed = self.W / norms vec_normed = vec / (np.linalg.norm(vec) + 1e-8) sims = normed @ vec_normed sims[self.word2idx[word]] = -1 top_idx = np.argsort(sims)[-topn:][::-1] return [(self.vocab[i], sims[i]) for i in top_idx] corpus = [ ['自然', '语言', '处理', '人工智能'], ['深度', '学习', '自然', '语言', '处理'], ['机器', '学习', '人工智能'], ['词', '向量', '语言', '表示'], ['文本', '分类', '自然', '语言'], ] model = SimpleGloVe(vector_size=20, alpha=0.75) model.fit(corpus, window=2, epochs=100, lr=0.05) print("\n=== 相似词 ===") for word in ['语言', '学习', '自然']: similar = model.most_similar(word, topn=3) print(f" '{word}': {[(w, f'{s:.3f}') for w, s in similar]}")