让机器读懂文字，从词频统计到情感理解
NLP（Natural Language Processing，自然语言处理）让计算机理解、生成和处理人类语言，是AI最活跃的领域之一。
| 任务 | 输入 | 输出 | 方法 |
|---|---|---|---|
| 情感分析 | 评论文本 | 正面/负面 | TF-IDF/BERT |
| 文本分类 | 新闻文章 | 类别标签 | CNN/Transformer |
| 命名实体识别 | 原始文本 | 实体标注 | CRF/BiLSTM-CRF |
| 机器翻译 | 源语言 | 目标语言 | Transformer |
| 文本生成 | 提示词 | 生成文本 | GPT/LLaMA |
| 问答系统 | 问题+上下文 | 答案 | RAG/BERT |
机器只能处理数字，NLP的第一步是把文本转换为数值表示。
import numpy as np
import re
# 文本预处理Pipeline
def tokenize(text):
"""英文分词: 小写 + 提取单词"""
return re.findall(r'\b\w+\b', text.lower())
def compute_tf(words):
"""词频TF: 每个词在文档中的频率"""
tf = {}
for w in words:
tf[w] = tf.get(w, 0) + 1
total = len(words)
return {w: c/total for w, c in tf.items()}
def compute_idf(documents):
"""逆文档频率IDF: 在所有文档中的稀有程度"""
n_docs = len(documents)
idf = {}
doc_freq = {}
for doc in documents:
unique_words = set(tokenize(doc))
for w in unique_words:
doc_freq[w] = doc_freq.get(w, 0) + 1
for w, df in doc_freq.items():
idf[w] = np.log((n_docs + 1) / (df + 1)) + 1 # 平滑IDF
return idf
# 示例文档
docs = [
"this movie is great and amazing",
"terrible movie waste of time",
"great acting wonderful story",
"bad movie boring and dull"
]
idf = compute_idf(docs)
print(f"词表大小: {len(idf)}")
# 计算第一个文档的TF-IDF
sample = docs[0]
tf = compute_tf(tokenize(sample))
top_tfidf = sorted([(w, tf.get(w,0)*idf.get(w,0))
for w in idf if w in tf], key=lambda x:-x[1])[:5]
print(f"示例: '{sample}'")
print(f"Top5 TF-IDF词: {[(w, f'{s:.3f}') for w,s in top_tfidf]}")
# 关键发现: "amazing"比"this"的TF-IDF更高
# 因为"amazing"只在1个文档出现(IDF高)，"this"在多个文档(IDF低)from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, classification_report
import numpy as np
np.random.seed(42)
# 构建情感数据集（模拟影评评论）
pos_templates = [
"this movie is great and amazing", "really enjoyed the film wonderful acting",
"excellent performance highly recommended", "beautiful cinematography loved every minute",
"fantastic story brilliant direction", "outstanding film one of the best",
"superb acting great screenplay", "wonderful movie truly inspiring",
"amazing visuals perfect casting", "delightful film heartwarming story",
]
neg_templates = [
"terrible movie waste of time", "awful film boring and dull",
"worst movie i have ever seen", "horrible acting bad screenplay",
"disappointing film not recommended", "poor performance weak storyline",
"bad movie terrible direction", "waste of money avoid this film",
"dreadful film painful to watch", "uninspired acting weak plot",
]
# 数据增强
def augment(templates, n=15):
result = []
for t in templates:
result.append(t)
words = t.split()
for _ in range(n // len(templates)):
idx = np.random.randint(0, len(words))
new_words = words.copy()
new_words.insert(idx, words[np.random.randint(0, len(words))])
result.append(' '.join(new_words))
return result
pos_texts = augment(pos_templates, 150)
neg_texts = augment(neg_templates, 150)
texts = pos_texts + neg_texts
labels = [1]*len(pos_texts) + [0]*len(neg_texts)
# TF-IDF向量化
X_train, X_test, y_train, y_test = train_test_split(
texts, labels, test_size=0.25, random_state=42)
vectorizer = TfidfVectorizer(max_features=300, stop_words='english')
X_train_tfidf = vectorizer.fit_transform(X_train)
X_test_tfidf = vectorizer.transform(X_test)
print(f"TF-IDF矩阵: {X_train_tfidf.shape}")
# 朴素贝叶斯分类
nb = MultinomialNB()
nb.fit(X_train_tfidf, y_train)
nb_acc = accuracy_score(y_test, nb.predict(X_test_tfidf))
# 逻辑回归分类
lr = LogisticRegression(random_state=42, max_iter=1000)
lr.fit(X_train_tfidf, y_train)
lr_acc = accuracy_score(y_test, lr.predict(X_test_tfidf))
print(f"朴素贝叶斯准确率: {nb_acc:.4f}")
print(f"逻辑回归准确率: {lr_acc:.4f}")# 从逻辑回归系数提取关键词
feature_names = vectorizer.get_feature_names_out()
lr_coefs = lr.coef_[0]
top_pos = [(feature_names[i], lr_coefs[i])
for i in np.argsort(lr_coefs)[-8:]][::-1]
top_neg = [(feature_names[i], lr_coefs[i])
for i in np.argsort(lr_coefs)[:8]]
print("正面关键词 (系数越大越正面):")
for w, c in top_pos:
bar = '█' * int(c * 3)
print(f" {w:>12}: {c:+.3f} {bar}")
print("负面关键词 (系数越小越负面):")
for w, c in top_neg:
bar = '█' * int(abs(c) * 3)
print(f" {w:>12}: {c:+.3f} {bar}")
# 关键词直觉:
# "great", "excellent", "brilliant" → 强正面信号
# "bad", "boring", "waste", "terrible" → 强负面信号
# 与人类直觉完全一致!