📖 第26课：文本摘要（抽取式）

import jieba from collections import Counter import math class ExtractiveSummarizer: def __init__(self): pass def sentence_tokenizer(self, text): import re sentences = re.split(r'[。！？\n]+', text) return [s.strip() for s in sentences if len(s.strip()) > 5] def score_tfidf(self, sentences): # 基于TF-IDF的句子评分 from sklearn.feature_extraction.text import TfidfVectorizer vectorizer = TfidfVectorizer(token_pattern=r'(?u)\b\w+\b') X = vectorizer.fit_transform([' '.join(jieba.cut(s)) for s in sentences]) scores = X.sum(axis=1).A1 return scores def score_position(self, sentences): # 位置评分：首尾句更重要 n = len(sentences) scores = [] for i in range(n): if i == 0: scores.append(1.0) elif i == n - 1: scores.append(0.8) else: scores.append(max(0.3, 1.0 - i/n)) return scores def score_length(self, sentences): # 长度评分：太短太长都不好 lengths = [len(s) for s in sentences] max_len = max(lengths) if lengths else 1 scores = [1.0 - abs(l/max_len - 0.6) for l in lengths] return scores def summarize(self, text, ratio=0.4): sentences = self.sentence_tokenizer(text) if len(sentences) <= 2: return text scores_tfidf = self.score_tfidf(sentences) scores_pos = self.score_position(sentences) scores_len = self.score_length(sentences) # 归一化 def normalize(arr): max_v = max(arr) if arr else 1 return [v/max_v for v in arr] scores_tfidf = normalize(scores_tfidf) scores_pos = normalize(scores_pos) scores_len = normalize(scores_len) # 综合评分 final_scores = [0.5*t + 0.3*p + 0.2*l for t, p, l in zip(scores_tfidf, scores_pos, scores_len)] # 选择top句子 n_select = max(1, int(len(sentences) * ratio)) top_indices = sorted(range(len(sentences)), key=lambda i: -final_scores[i])[:n_select] top_indices.sort() # 保持原文顺序 summary = ''.join([sentences[i] for i in top_indices]) return summary, [(sentences[i], final_scores[i]) for i in range(len(sentences))] text = "自然语言处理是人工智能的重要分支。深度学习技术推动了自然语言处理的快速发展。词向量表示和预训练语言模型成为NLP的核心技术。文本分类、情感分析、机器翻译等任务取得了显著进展。大规模预训练模型如BERT和GPT改变了NLP的研究范式。中国的研究人员在自然语言处理领域也做出了重要贡献。未来，多模态和跨语言NLP将成为研究热点。" summarizer = ExtractiveSummarizer() summary, scores = summarizer.summarize(text, ratio=0.4) print("=== 抽取式摘要 ===") print(f"原文({len(text)}字): {text[:50]}...") print(f"摘要({len(summary)}字): {summary}") print("\n=== 句子评分 ===") for sent, score in scores: bar = '█' * int(score * 20) print(f" [{bar}] {score:.3f} | {sent[:20]}...")

输入文本 → 文本分段 → 句子切分 → 句子向量化
    → 句子评分(TF-IDF+位置+新颖度) → 冗余去除
    → 句子选择(优化问题) → 排序 → 摘要输出