📖 第29课：信息检索

import jieba from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics.pairwise import cosine_similarity import numpy as np class SearchEngine: def __init__(self): self.documents = [] self.vectorizer = TfidfVectorizer(token_pattern=r'(?u)\b\w+\b', max_features=1000) self.doc_matrix = None def index(self, documents): self.documents = documents doc_texts = [' '.join(jieba.cut(doc)) for doc in documents] self.doc_matrix = self.vectorizer.fit_transform(doc_texts) def search(self, query, top_k=5): query_vec = self.vectorizer.transform([' '.join(jieba.cut(query))]) scores = cosine_similarity(query_vec, self.doc_matrix)[0] top_indices = np.argsort(scores)[-top_k:][::-1] results = [] for idx in top_indices: if scores[idx] > 0: results.append({ 'doc_id': idx, 'content': self.documents[idx], 'score': scores[idx] }) return results # 构建文档库 documents = [ "自然语言处理是人工智能的重要分支，研究计算机理解和生成人类语言的技术", "深度学习使用多层神经网络从数据中学习特征表示，在图像识别和语音识别领域取得突破", "词向量是将词映射到低维连续空间的技术，使语义相似的词在空间中接近", "BERT是Google提出的预训练语言模型，通过双向Transformer学习上下文化表示", "知识图谱以实体和关系为核心，构建结构化的语义网络", "机器翻译使用序列到序列模型将一种语言翻译为另一种语言", "情感分析判断文本的情感倾向，在电商评论和社交媒体分析中应用广泛", "信息检索从大量文档中找到与查询最相关的文档", "命名实体识别从文本中识别人名、地名、机构名等实体", "文本摘要自动生成文本的简洁版本，分为抽取式和生成式两种方法", ] engine = SearchEngine() engine.index(documents) queries = ["什么是深度学习", "词的表示方法", "文本情感分析", "知识图谱是什么"] for query in queries: results = engine.search(query, top_k=3) print(f"查询: {query}") for r in results: print(f" [{r['score']:.4f}] {r['content'][:40]}...") print()

class BM25: def __init__(self, k1=1.5, b=0.75): self.k1 = k1 self.b = b self.doc_count = 0 self.avg_dl = 0 self.doc_lens = [] self.df = {} self.doc_terms = [] def fit(self, documents): import jieba self.doc_count = len(documents) self.doc_terms = [jieba.lcut(doc) for doc in documents] self.doc_lens = [len(terms) for terms in self.doc_terms] self.avg_dl = sum(self.doc_lens) / len(self.doc_lens) for terms in self.doc_terms: for term in set(terms): self.df[term] = self.df.get(term, 0) + 1 def score(self, query, doc_idx): import math query_terms = jieba.lcut(query) doc = self.doc_terms[doc_idx] dl = self.doc_lens[doc_idx] tf_dict = {} for t in doc: tf_dict[t] = tf_dict.get(t, 0) + 1 score = 0 for term in query_terms: if term not in tf_dict: continue tf = tf_dict[term] df = self.df.get(term, 0) idf = math.log((self.doc_count - df + 0.5) / (df + 0.5) + 1) numerator = tf * (self.k1 + 1) denominator = tf + self.k1 * (1 - self.b + self.b * dl / self.avg_dl) score += idf * numerator / denominator return score def search(self, query, top_k=5): scores = [(i, self.score(query, i)) for i in range(self.doc_count)] scores.sort(key=lambda x: -x[1]) return scores[:top_k] bm25 = BM25() bm25.fit(documents) print("=== BM25检索 ===") for query in ["什么是深度学习", "词的表示方法", "情感分析技术"]: results = bm25.search(query, top_k=3) print(f"查询: {query}") for idx, score in results: print(f" [{score:.4f}] {documents[idx][:40]}...") print()