🎯 第10课：实体链接

import re from collections import defaultdict from typing import List, Dict, Tuple, Optional class KnowledgeBase: ">>>简易知识库（模拟）""" def __init__(self): self.entities = {} # {entity_id: {name, type, aliases, desc, facts}} self.name_index = defaultdict(list) # {name: [entity_ids]} def add_entity(self, entity_id, name, etype, aliases=None, desc="", facts=None): self.entities[entity_id] = { "name": name, "type": etype, "aliases": aliases or [], "desc": desc, "facts": facts or {} } self.name_index[name].append(entity_id) for alias in (aliases or []): self.name_index[alias].append(entity_id) def search(self, mention): """根据指称搜索候选实体""" return self.name_index.get(mention, []) def get_entity(self, entity_id): return self.entities.get(entity_id) class EntityLinker: """实体链接系统""" def __init__(self, kb): self.kb = kb self.popularity = {} # {entity_id: popularity_score} self.type_weights = {"PER": 1.0, "LOC": 0.8, "ORG": 0.9, "WORK": 0.7} def set_popularity(self, entity_id, score): self.popularity[entity_id] = score def _context_similarity(self, mention_context, entity_desc): """计算上下文与实体描述的相似度（词重叠）""" ctx_words = set(mention_context) desc_words = set(entity_desc) if not ctx_words or not desc_words: return 0.0 overlap = ctx_words & desc_words return len(overlap) / len(ctx_words | desc_words) def link(self, mention, mention_type=None, context=""): """将指称链接到知识库实体""" candidates = self.kb.search(mention) if not candidates: return {"mention": mention, "entity": None, "status": "NIL", "confidence": 0.0} if len(candidates) == 1: return {"mention": mention, "entity": candidates[0], "status": "UNAMBIG", "confidence": 1.0} # 多候选：消歧打分 scores = {} for eid in candidates: entity = self.kb.get_entity(eid) score = 0.0 # 流行度先验 score += self.popularity.get(eid, 0.5) * 0.3 # 类型匹配 if mention_type and entity["type"] == mention_type: score += 0.3 # 上下文相似度 if context: sim = self._context_similarity(context, entity["desc"]) score += sim * 0.4 scores[eid] = score best_eid = max(scores, key=scores.get) best_score = scores[best_eid] return { "mention": mention, "entity": best_eid, "status": "DISAMBIG", "confidence": best_score, "candidates": len(candidates) } def batch_link(self, mentions_with_context): """批量链接""" results = [] for mention, mtype, ctx in mentions_with_context: result = self.link(mention, mtype, ctx) results.append(result) return results # ========== 构建实体链接系统 ========== kb = KnowledgeBase() # 两个"李白" kb.add_entity("libai_poet", "李白", "PER", aliases=["李太白", "诗仙", "青莲居士"], desc="唐代伟大的浪漫主义诗人 被誉为诗仙 与杜甫并称李杜", facts={"朝代": "唐", "风格": "浪漫主义"}) kb.add_entity("libai_director", "李白", "PER", aliases=[], desc="现代导演 电影制作人", facts={"职业": "导演"}) # 两个"北京大学" kb.add_entity("pku_university", "北京大学", "ORG", aliases=["北大", "燕园"], desc="中国顶尖综合性大学 位于北京海淀区 创建于1898年") kb.add_entity("pku_press", "北京大学出版社", "ORG", aliases=["北大出版社"], desc="北京大学下属出版社 学术出版") # 其他实体 kb.add_entity("luxun", "鲁迅", "PER", aliases=["周树人", "鲁迅先生"], desc="中国现代文学的奠基人 作家 文学革命") kb.add_entity("beijing_city", "北京", "LOC", desc="中国首都 政治中心 文化中心") # 创建链接器 linker = EntityLinker(kb) linker.set_popularity("libai_poet", 0.95) linker.set_popularity("libai_director", 0.15) linker.set_popularity("pku_university", 0.9) # 测试消歧 test_cases = [ ("李白", "PER", "唐代诗人杜甫与李白并称李杜"), ("李白", "PER", "导演李白的新片即将上映"), ("北大", "ORG", "他考上了北大 位于海淀区"), ("鲁迅", "PER", "文学革命中的鲁迅"), ("王阳明", "PER", "心学大师"), # 不在KB中 ] print("=== 实体链接消歧 ===" for mention, mtype, ctx in test_cases: result = linker.link(mention, mtype, ctx) entity = kb.get_entity(result["entity"]) if result["entity"] else None if entity: print(f" {mention} → {entity['name']}({result['entity']}) [{result['status']}] conf={result['confidence']:.2f}") else: print(f" {mention} → NIL (不在知识库中)")