🏷️ 第6课：命名实体识别

import re from collections import defaultdict class RuleBasedNER: """基于规则和词典的命名实体识别器""" def __init__(self): self.dictionaries = defaultdict(set) # {entity_type: {entities}} self.patterns = [] # [(regex, entity_type)] self.prefixes = defaultdict(list) # {entity_type: [prefixes]} def add_dictionary(self, entity_type, entities): """添加实体词典""" for e in entities: self.dictionaries[entity_type].add(e) def add_pattern(self, regex, entity_type): ">>>添加正则模式""" self.patterns.append((re.compile(regex), entity_type)) def add_prefix(self, entity_type, prefixes): """添加实体前缀（如'省'、'市'）""" self.prefixes[entity_type].extend(prefixes) def recognize(self, text): """识别文本中的命名实体，返回 [(实体, 类型, 起始位置, 结束位置)]""" results = [] # 1. 词典匹配（最长匹配优先） for entity_type, entities in self.dictionaries.items(): for entity in sorted(entities, key=len, reverse=True): start = 0 while True: idx = text.find(entity, start) if idx == -1: break # 检查是否已被更长的实体覆盖 if not any(s <= idx < e or s < idx + len(entity) <= e for _, _, s, e in results): results.append((entity, entity_type, idx, idx + len(entity))) start = idx + 1 # 2. 正则模式匹配 for pattern, entity_type in self.patterns: for match in pattern.finditer(text): entity = match.group() if not any(s <= match.start() < e or s < match.end() <= e for _, _, s, e in results): results.append((entity, entity_type, match.start(), match.end())) # 3. 前缀模式匹配 for entity_type, prefixes in self.prefixes.items(): for prefix in prefixes: pattern = re.compile(r'[一-鿿]{1,4}' + re.escape(prefix)) for match in pattern.finditer(text): entity = match.group() if not any(s <= match.start() < e or s < match.end() <= e for _, _, s, e in results): results.append((entity, entity_type, match.start(), match.end())) results.sort(key=lambda x: x[2]) return results def format_results(self, text, results): """格式化输出识别结果""" lines = [] for entity, etype, start, end in results: lines.append(f" [{etype}] {entity} (位置: {start}-{end})") return " ".join(lines) # ========== 构建中文NER系统 ========== ner = RuleBasedNER() # 词典 ner.add_dictionary("PER", ["鲁迅", "老舍", "徐志摩", "巴金", "茅盾", "毛泽东"]) ner.add_dictionary("LOC", ["绍兴", "北京", "上海", "海宁", "浙江", "浙江省", "中国"]) ner.add_dictionary("ORG", ["北京大学", "清华大学", "浙江大学", "中华文学社"]) ner.add_dictionary("WORK", ["呐喊", "彷徨", "骆驼祥子", "再别康桥", "家", "子夜"]) # 正则模式 ner.add_pattern(r'\d{4}年', "TIME") ner.add_pattern(r'\d{1,2}月\d{1,2}日', "TIME") ner.add_pattern(r'《[^》]+》', "WORK") # 前缀模式 ner.add_prefix("LOC", ["省", "市", "区", "县", "镇"]) ner.add_prefix("ORG", ["大学", "学院", "公司", "集团"]) # 测试 test_texts = [ "鲁迅出生于浙江省绍兴，1923年出版了《呐喊》。", "老舍在北京大学任教，创作了骆驼祥子和茶馆。", "徐志摩是海宁人，1931年11月19日因飞机失事遇难。", "巴金原名李尧棠，出生于四川省成都市。", ] for text in test_texts: results = ner.recognize(text) print(f"原文: {text}") print(ner.format_results(text, results)) print()

class BIOTagger: """BIO序列标注与解码""" def encode(self, text, entities): """将文本和实体标注转换为BIO标签序列""" tags = ["O"] * len(text) for entity, etype, start, end in entities: tags[start] = f"B-{etype}" for i in range(start + 1, end): tags[i] = f"I-{etype}" return list(zip(text, tags)) def decode(self, tagged_sequence): """从BIO标签序列解码出实体""" entities = [] current_entity = "" current_type = None start = None for i, (char, tag) in enumerate(tagged_sequence): if tag.startswith("B-"): if current_entity: entities.append((current_entity, current_type, start, i)) current_type = tag[2:] current_entity = char start = i elif tag.startswith("I-") and current_type == tag[2]: current_entity += char else: if current_entity: entities.append((current_entity, current_type, start, i)) current_entity = "" current_type = None start = None if current_entity: entities.append((current_entity, current_type, start, len(tagged_sequence))) return entities # BIO标注演示 tagger = BIOTagger() text = "鲁迅出生于绍兴" entities = [("鲁迅", "PER", 0, 2), ("绍兴", "LOC", 4, 6)] tagged = tagger.encode(text, entities) print("=== BIO标注 ===") for char, tag in tagged: print(f" {char} → {tag}") # 解码验证 decoded = tagger.decode(tagged) print(" === BIO解码 ===") for entity, etype, s, e in decoded: print(f" [{etype}] {entity} ({s}-{e})")

def evaluate_ner(gold, pred): """评估NER结果 Args: gold: 标注正确的实体集合 {(entity, type, start, end)} pred: 预测的实体集合 """ gold_set = set((e, t, s, en) for e, t, s, en in gold) pred_set = set((e, t, s, en) for e, t, s, en in pred) tp = len(gold_set & pred_set) fp = len(pred_set - gold_set) fn = len(gold_set - pred_set) precision = tp / (tp + fp) if tp + fp > 0 else 0 recall = tp / (tp + fn) if tp + fn > 0 else 0 f1 = 2 * precision * recall / (precision + recall) if precision + recall > 0 else 0 return {"Precision": f"{precision:.4f}", "Recall": f"{recall:.4f}", "F1": f"{f1:.4f}"} # 评估示例 gold_entities = [("鲁迅", "PER", 0, 2), ("绍兴", "LOC", 4, 6), ("北京大学", "ORG", 8, 12)] pred_entities = [("鲁迅", "PER", 0, 2), ("绍兴", "LOC", 4, 6), ("北京", "LOC", 8, 10)] print("=== NER评估 ===") metrics = evaluate_ner(gold_entities, pred_entities) for k, v in metrics.items(): print(f" {k}: {v}")