📖 第17课：文本特征工程

from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.linear_model import LogisticRegression from sklearn.model_selection import cross_val_score import jieba import numpy as np # 原始文本 texts_raw = [ "这部电影非常好看，剧情精彩，演员演技出色", "太差了，浪费时间，剧情无聊", "还行吧，一般般，中规中矩", "强烈推荐，超出预期，非常满意", "演技尴尬，令人失望，不推荐", "音乐优美，画面精致，经典之作", "烂片无疑，别去看，后悔购票", "温馨治愈，心情愉悦，百看不厌", ] labels = ['pos', 'neg', 'neu', 'pos', 'neg', 'pos', 'neg', 'pos'] # 特征工程策略1: 字符级n-gram vec_char = TfidfVectorizer(analyzer='char', ngram_range=(2,4), max_features=500) X_char = vec_char.fit_transform(texts_raw) scores_char = cross_val_score(LogisticRegression(max_iter=1000, random_state=42), X_char, labels, cv=3) print(f"字符n-gram: {scores_char.mean():.4f} (±{scores_char.std():.4f})") # 特征工程策略2: 词级TF-IDF texts_seg = [' '.join(jieba.cut(t)) for t in texts_raw] vec_word = TfidfVectorizer(token_pattern=r'(?u)\b\w+\b', max_features=200) X_word = vec_word.fit_transform(texts_seg) scores_word = cross_val_score(LogisticRegression(max_iter=1000, random_state=42), X_word, labels, cv=3) print(f"词级TF-IDF: {scores_word.mean():.4f} (±{scores_word.std():.4f})") # 特征工程策略3: 混合特征 vec_mix = TfidfVectorizer(token_pattern=r'(?u)\b\w+\b', ngram_range=(1,2), max_features=500) X_mix = vec_mix.fit_transform(texts_seg) scores_mix = cross_val_score(LogisticRegression(max_iter=1000, random_state=42), X_mix, labels, cv=3) print(f"混合1-2gram: {scores_mix.mean():.4f} (±{scores_mix.std():.4f})") # 自定义特征 def extract_features(text): features = {} features['len'] = len(text) features['punct_count'] = sum(1 for c in text if c in '！？。，') features['exclaim'] = text.count('！') features['question'] = text.count('？') features['comma'] = text.count('，') return features feat_dicts = [extract_features(t) for t in texts_raw] from sklearn.feature_extraction import DictVectorizer dv = DictVectorizer() X_custom = dv.fit_transform(feat_dicts) scores_custom = cross_val_score(LogisticRegression(max_iter=1000, random_state=42), X_custom, labels, cv=3) print(f"自定义特征: {scores_custom.mean():.4f} (±{scores_custom.std():.4f})") print("\n=== 特征工程总结 ===") print(f" 字符n-gram: {scores_char.mean():.4f}") print(f" 词级TF-IDF: {scores_word.mean():.4f}") print(f" 混合n-gram: {scores_mix.mean():.4f}") print(f" 自定义特征: {scores_custom.mean():.4f}")