📖 第27课：问答系统（检索式）

import jieba from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics.pairwise import cosine_similarity import numpy as np class RetrievalQA: def __init__(self): self.knowledge_base = [] self.vectorizer = TfidfVectorizer(token_pattern=r'(?u)\b\w+\b') self.tfidf_matrix = None def add_knowledge(self, qa_pairs): self.knowledge_base = qa_pairs questions = [' '.join(jieba.cut(q)) for q, _ in qa_pairs] self.tfidf_matrix = self.vectorizer.fit_transform(questions) def search(self, query, top_k=3): query_vec = self.vectorizer.transform([' '.join(jieba.cut(query))]) scores = cosine_similarity(query_vec, self.tfidf_matrix)[0] top_indices = np.argsort(scores)[-top_k:][::-1] results = [] for idx in top_indices: q, a = self.knowledge_base[idx] results.append({'question': q, 'answer': a, 'score': scores[idx]}) return results def answer(self, query, threshold=0.1): results = self.search(query, top_k=1) if results and results[0]['score'] >= threshold: return results[0]['answer'], results[0]['score'] return "抱歉，我不确定这个问题的答案。", 0.0 qa_pairs = [ ("什么是自然语言处理", "自然语言处理(NLP)是计算机科学与人工智能的交叉领域，研究如何让计算机理解和生成人类语言。"), ("什么是深度学习", "深度学习是机器学习的分支，使用多层神经网络从数据中学习特征表示。"), ("什么是词向量", "词向量是词的分布式数值表示，将词映射到低维连续空间，使语义相似的词在空间中接近。"), ("什么是BERT", "BERT是Google提出的预训练语言模型，通过双向Transformer编码器学习上下文化词表示。"), ("什么是TF-IDF", "TF-IDF是一种统计方法，衡量词对文档的重要性，结合词频和逆文档频率。"), ("什么是情感分析", "情感分析是NLP任务，自动判断文本的情感倾向（正面、负面或中性）。"), ("什么是知识图谱", "知识图谱是结构化的知识表示，以实体为节点、关系为边构建语义网络。"), ("什么是机器翻译", "机器翻译是利用计算机自动将一种语言翻译成另一种语言的技术。"), ] qa = RetrievalQA() qa.add_knowledge(qa_pairs) queries = [ "NLP是什么", "词嵌入是什么意思", "怎么衡量词的重要性", "什么是神经网络", ] print("=== 检索式问答系统 ===") for q in queries: answer, score = qa.answer(q) print(f" 问: {q}") print(f" 答: {answer} (相似度: {score:.4f})") print()

import jieba from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics.pairwise import cosine_similarity import numpy as np class AdvancedQA(RetrievalQA): def keyword_search(self, query, top_k=3): # 关键词匹配补充 query_words = set(jieba.cut(query)) results = [] for i, (q, a) in enumerate(self.knowledge_base): kb_words = set(jieba.cut(q)) overlap = len(query_words & kb_words) / max(len(query_words | kb_words), 1) results.append((i, overlap)) results.sort(key=lambda x: -x[1]) return [(self.knowledge_base[i][0], self.knowledge_base[i][1], s) for i, s in results[:top_k]] def hybrid_search(self, query, alpha=0.7, top_k=3): # 混合检索：TF-IDF + 关键词 tfidf_results = self.search(query, top_k=len(self.knowledge_base)) keyword_results = self.keyword_search(query, top_k=len(self.knowledge_base)) scores = {} for r in tfidf_results: key = r['question'] scores[key] = scores.get(key, 0) + alpha * r['score'] for q, a, s in keyword_results: scores[q] = scores.get(q, 0) + (1 - alpha) * s ranked = sorted(scores.items(), key=lambda x: -x[1])[:top_k] return [(q, dict(self.knowledge_base)[q], s) for q, s in ranked] qa = AdvancedQA() qa.add_knowledge(qa_pairs) print("=== 混合检索问答 ===") queries = ["深度学习是什么", "情感分析怎么做的"] for q in queries: results = qa.hybrid_search(q, top_k=2) print(f" 问: {q}") for question, answer, score in results: print(f" 匹配: {question} (分数: {score:.4f})") print(f" 答: {answer[:50]}...") print() # qa_pairs defined above qa_pairs = [ ("什么是自然语言处理", "自然语言处理(NLP)是计算机科学与人工智能的交叉领域"), ("什么是深度学习", "深度学习是机器学习的分支，使用多层神经网络"), ("什么是词向量", "词向量是词的分布式数值表示"), ("什么是BERT", "BERT是Google提出的预训练语言模型"), ("什么是TF-IDF", "TF-IDF衡量词对文档的重要性"), ("什么是情感分析", "情感分析自动判断文本情感倾向"), ("什么是知识图谱", "知识图谱是结构化的知识表示"), ("什么是机器翻译", "机器翻译自动将一种语言翻译成另一种语言"), ]