📖 第30课：毕业项目：智能文档分析系统

原始文本
    ↓
[文本预处理] → 清洗、分词、词性标注
    ↓
[命名实体识别] → 人名、地名、机构
    ↓
[关键词提取] → TF-IDF/TextRank
    ↓
[关系抽取] → 实体间语义关系
    ↓
[摘要生成] → 抽取式摘要
    ↓
分析报告

import jieba import jieba.posseg as pseg import jieba.analyse # 需要单独导入analyse子模块 from collections import Counter, defaultdict import re import math class DocumentAnalyzer: def __init__(self): self.entities = [] self.keywords = [] self.summary = "" self.relations = [] self.stats = {} def analyze(self, text): # 1. 基础统计 self.stats = { 'char_count': len(text), 'sentence_count': len(re.split(r'[。！？]', text)), 'word_count': len(jieba.lcut(text)), } # 2. 命名实体识别 words = list(pseg.cut(text)) type_map = {'nr': '人名', 'ns': '地名', 'nt': '机构', 'nz': '专名'} self.entities = [(w.word, type_map.get(w.flag, w.flag)) for w in words if w.flag in ('nr', 'ns', 'nt', 'nz')] # 3. 关键词提取 self.keywords = jieba.analyse.extract_tags(text, topK=10, withWeight=True) # 4. 简单关系抽取 self._extract_relations(text) # 5. 摘要 self._extract_summary(text) return self def _extract_relations(self, text): sentences = re.split(r'[。！？]', text) patterns = [ (r'(.+?)是(.+?)的(.+)', '属性'), (r'(.+?)位于(.+)', '位置'), (r'(.+?)(创建|成立|创办)了?(.+)', '创建'), ] for sent in sentences: for pattern, rel_type in patterns: match = re.search(pattern, sent) if match: groups = match.groups() if len(groups) >= 2: self.relations.append((groups[0].strip(), rel_type, groups[-1].strip())) def _extract_summary(self, text): sentences = re.split(r'[。！？]', text) sentences = [s.strip() for s in sentences if len(s.strip()) > 10] if not sentences: self.summary = text[:100] return word_freq = Counter(jieba.lcut(text)) scores = [] for sent in sentences: words = jieba.lcut(sent) score = sum(word_freq.get(w, 0) for w in words) / max(len(words), 1) scores.append(score) n_select = max(1, len(sentences) // 3) top_indices = sorted(range(len(sentences)), key=lambda i: -scores[i])[:n_select] top_indices.sort() self.summary = '。'.join([sentences[i] for i in top_indices]) + '。' def report(self): print("=" * 60) print("📋 智能文档分析报告") print("=" * 60) print(f"\n📊 基础统计:") for k, v in self.stats.items(): print(f" {k}: {v}") print(f"\n🏷️ 命名实体:") entity_types = defaultdict(list) for name, etype in self.entities: entity_types[etype].append(name) for etype, names in entity_types.items(): print(f" {etype}: {list(set(names))}") print(f"\n🔑 关键词:") for word, weight in self.keywords[:8]: print(f" {word}: {weight:.4f}") print(f"\n🔗 关系:") for subj, rel, obj in self.relations: print(f" {subj} --[{rel}]--> {obj}") print(f"\n📝 摘要:") print(f" {self.summary}") # 测试 text = ( '阿里巴巴集团是中国最大的电子商务公司。' '马云于1999年在杭州创建了阿里巴巴。' '阿里巴巴旗下拥有淘宝、天猫等电商平台。' '张勇目前担任阿里巴巴的CEO。' '阿里巴巴在云计算和人工智能领域也有重要布局。' '阿里巴巴的总部位于浙江省杭州市。' '近年来，阿里巴巴在自然语言处理和计算机视觉等AI技术上投入了大量资源。' ) analyzer = DocumentAnalyzer() analyzer.analyze(text) analyzer.report()

class DocumentComparator: def __init__(self): pass def compare(self, text1, text2): from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics.pairwise import cosine_similarity import jieba texts = [' '.join(jieba.cut(text1)), ' '.join(jieba.cut(text2))] vectorizer = TfidfVectorizer(token_pattern=r'(?u)\b\w+\b') matrix = vectorizer.fit_transform(texts) similarity = cosine_similarity(matrix[0:1], matrix[1:2])[0][0] # 关键词重叠 kw1 = set(jieba.analyse.extract_tags(text1, topK=10)) kw2 = set(jieba.analyse.extract_tags(text2, topK=10)) overlap = kw1 & kw2 return { 'similarity': similarity, 'keywords_1': kw1, 'keywords_2': kw2, 'keyword_overlap': overlap, 'overlap_ratio': len(overlap) / max(len(kw1 | kw2), 1), } # 文档对比 text1 = "阿里巴巴是中国最大的电子商务公司，马云于1999年在杭州创建。阿里巴巴在云计算和AI领域有重要布局。" text2 = "腾讯是中国领先的互联网公司，马化腾于1998年在深圳创建。腾讯在社交和游戏领域占据主导地位。" comparator = DocumentComparator() result = comparator.compare(text1, text2) print("=== 文档对比分析 ===") print(f"语义相似度: {result['similarity']:.4f}") print(f"文档1关键词: {result['keywords_1']}") print(f"文档2关键词: {result['keywords_2']}") print(f"关键词重叠: {result['keyword_overlap']}") print(f"重叠比例: {result['overlap_ratio']:.4f}")

API层(FastAPI) → 分析服务 → NLP Pipeline
                    ↓
              缓存层(Redis) → 向量库(Milvus)
                    ↓
              存储层(PostgreSQL + Neo4j)