📖 第25课：关键词提取

import jieba import jieba.analyse from collections import Counter import math text = "自然语言处理是人工智能的重要分支。深度学习技术推动了自然语言处理的快速发展。词向量表示和预训练语言模型成为NLP的核心技术。文本分类、情感分析、机器翻译等任务取得了显著进展。大规模预训练模型如BERT和GPT改变了NLP的研究范式。" # 方法1: TF-IDF关键词提取 tfidf_keywords = jieba.analyse.extract_tags(text, topK=10, withWeight=True) print("=== TF-IDF关键词 ===") for word, weight in tfidf_keywords: print(f" {word}: {weight:.4f}") # 方法2: TextRank关键词提取 textrank_keywords = jieba.analyse.textrank(text, topK=10, withWeight=True) print("\n=== TextRank关键词 ===") for word, weight in textrank_keywords: print(f" {word}: {weight:.4f}") # 方法3: 手动实现TextRank import numpy as np def manual_textrank(words, window=4, d=0.85, iterations=30): word_set = list(set(words)) n = len(word_set) word2idx = {w: i for i, w in enumerate(word_set)} # 构建共现图 graph = np.zeros((n, n)) for i in range(len(words)): for j in range(max(0, i-window), min(len(words), i+window+1)): if i != j: a, b = word2idx[words[i]], word2idx[words[j]] graph[a][b] += 1 # PageRank迭代 scores = np.ones(n) / n for _ in range(iterations): new_scores = (1 - d) / n + d * graph.T @ scores / (graph.sum(axis=0, keepdims=True) + 1e-8) scores = new_scores ranked = [(word_set[i], scores[i]) for i in range(n)] ranked.sort(key=lambda x: -x[1]) return ranked words = jieba.lcut(text) keywords = manual_textrank(words, window=4) print("\n=== 手动TextRank ===") for word, score in keywords[:10]: print(f" {word}: {score:.6f}")

import jieba from collections import Counter import math # RAKE算法实现 class RAKE: def __init__(self, stop_words=None): if stop_words is None: self.stop_words = {'的', '了', '在', '是', '我', '有', '和', '就', '不', '人', '都', '一', '一个', '上', '也', '很', '到', '说', '要', '去', '你', '会', '着', '没有', '看', '好', '自己', '这'} else: self.stop_words = stop_words def extract_keywords(self, text, topn=10): words = jieba.lcut(text) # 分割为候选关键词 candidates = [] current = [] for word in words: if word in self.stop_words or len(word.strip()) == 0: if current: candidates.append(current) current = [] else: current.append(word) if current: candidates.append(current) # 计算词频和共现度 word_freq = Counter() word_degree = Counter() for phrase in candidates: degree = len(phrase) - 1 for word in phrase: word_freq[word] += 1 word_degree[word] += degree # 计算 score = degree / frequency word_score = {} for word in word_freq: word_score[word] = (word_degree[word] + word_freq[word]) / word_freq[word] # 短语得分 phrase_scores = {} for phrase in candidates: score = sum(word_score.get(w, 0) for w in phrase) phrase_scores[' '.join(phrase)] = score ranked = sorted(phrase_scores.items(), key=lambda x: -x[1]) return ranked[:topn] rake = RAKE() text = "自然语言处理是人工智能的重要分支。深度学习推动了自然语言处理的发展。预训练语言模型改变了NLP的研究范式。词向量表示技术是深度学习在NLP中的基础应用。" keywords = rake.extract_keywords(text, topn=10) print("=== RAKE关键词提取 ===") for phrase, score in keywords: print(f" {phrase}: {score:.4f}")