🐍 第17课：网页爬虫

—— BeautifulSoup 数据提取

🏆 BeautifulSoup提取数据
✅ Python验证通过

📌 本课目标

1️⃣ BeautifulSoup 基础

from bs4 import BeautifulSoup  # pip install beautifulsoup4

html = """
<html>
  <head><title>测试页面</title></head>
  <body>
    <h1 class="title">欢迎</h1>
    <div id="content">
      <ul class="list">
        <li>项目1</li>
        <li>项目2</li>
        <li>项目3</li>
      </ul>
      <a href="https://example.com" target="_blank">链接</a>
    </div>
  </body>
</html>
"""

soup = BeautifulSoup(html, "html.parser")

# 获取标签
print(soup.title.string)          # 测试页面
print(soup.h1.string)             # 欢迎
print(soup.find("h1").string)     # 欢迎

# 获取属性
link = soup.find("a")
print(link["href"])              # https://example.com
print(link["target"])           # _blank
print(link.get("href"))           # 安全获取，不存在返回 None

# 获取文本
print(soup.h1.text)               # 欢迎
print(soup.h1.get_text(strip=True))  # 去除空白

2️⃣ 查找元素

from bs4 import BeautifulSoup

# find - 找第一个
soup.find("div")                         # 标签名
soup.find("div", id="content")              # 标签+id
soup.find("h1", class_="title")            # 标签+class（注意class_）
soup.find("a", attrs={"target": "_blank"})  # 任意属性

# find_all - 找所有
soup.find_all("li")              # 所有 li
soup.find_all("li", limit=2)       # 最多2个
soup.find_all(["h1", "h2", "h3"])   # 多种标签

# CSS 选择器（推荐！更灵活）
soup.select(".title")             # class
soup.select("#content")             # id
soup.select("ul.list li")         # 后代
soup.select("ul.list > li")        # 直接子元素
soup.select("a[href]")             # 有 href 属性的 a
soup.select("a[target='_blank']")  # 属性值匹配
soup.select("li:nth-child(2)")     # 第二个 li

# select_one - 找第一个（等于 find）
soup.select_one("h1.title")

# 遍历
for li in soup.select("ul.list li"):
    print(li.get_text(strip=True))

3️⃣ 实战：提取结构化数据

from bs4 import BeautifulSoup

html = """
<table class="data-table">
  <thead>
    <tr><th>姓名</th><th>年龄</th><th>城市</th></tr>
  </thead>
  <tbody>
    <tr><td>张三</td><td>28</td><td>北京</td></tr>
    <tr><td>李四</td><td>32</td><td>上海</td></tr>
    <tr><td>王五</td><td>25</td><td>深圳</td></tr>
  </tbody>
</table>
"""

def extract_table(soup) -> list[dict]:
    """提取 HTML 表格数据"""
    # 获取表头
    headers = [th.get_text(strip=True)
               for th in soup.select("table.data-table thead th")]

    # 获取数据行
    rows = []
    for tr in soup.select("table.data-table tbody tr"):
        cells = [td.get_text(strip=True) for td in tr.select("td")]
        rows.append(dict(zip(headers, cells)))

    return rows

soup = BeautifulSoup(html, "html.parser")
data = extract_table(soup)
# [{'姓名': '张三', '年龄': '28', '城市': '北京'}, ...]

4️⃣ 爬虫礼仪

import requests
import time
from pathlib import Path

# 1. 遵守 robots.txt
# 2. 控制请求频率
# 3. 设置 User-Agent
# 4. 尊重网站资源

class PoliteScraper:
    """礼貌的爬虫"""

    def __init__(self, delay: float = 1.0):
        self.session = requests.Session()
        self.session.headers.update({
            "User-Agent": "MyBot/1.0 (Educational Purpose)",
            "Accept": "text/html",
        })
        self.delay = delay
        self.last_request = 0

    def fetch(self, url: str) -> str:
        """获取页面（自动限速）"""
        import time
        elapsed = time.time() - self.last_request
        if elapsed < self.delay:
            time.sleep(self.delay - elapsed)

        resp = self.session.get(url, timeout=10)
        resp.raise_for_status()
        self.last_request = time.time()
        return resp.text

# 缓存已抓取页面（避免重复请求）
def cached_fetch(url: str, cache_dir: str = ".cache") -> str:
    """带缓存的页面获取"""
    import hashlib
    cache_path = Path(cache_dir) / hashlib.md5(url.encode()).hexdigest()
    if cache_path.exists():
        return cache_path.read_text(encoding="utf-8")

    resp = requests.get(url, timeout=10,
                        headers={"User-Agent": "MyBot/1.0"})
    resp.raise_for_status()
    cache_path.parent.mkdir(parents=True, exist_ok=True)
    cache_path.write_text(resp.text, encoding="utf-8")
    return resp.text

5️⃣ 验证脚本

#!/usr/bin/env python3
"""第17课 网页爬虫验证"""
from bs4 import BeautifulSoup

def test_basic_parsing():
    """基本解析测试"""
    html = "<html><head><title>测试</title></head><body><h1>Hello</h1></body></html>"
    soup = BeautifulSoup(html, "html.parser")
    assert soup.title.string == "测试"
    assert soup.h1.string == "Hello"
    print("✅ 基本解析测试通过")

def test_find_elements():
    """元素查找测试"""
    html = """
    <div id="main">
      <h1 class="title">标题</h1>
      <ul class="list">
        <li>A</li><li>B</li><li>C</li>
      </ul>
      <a href="https://example.com">链接</a>
    </div>
    """
    soup = BeautifulSoup(html, "html.parser")
    assert soup.find("h1", class_="title").string == "标题"
    assert soup.find("div", id="main") is not None
    assert soup.find("a")["href"] == "https://example.com"
    lis = soup.find_all("li")
    assert len(lis) == 3
    print("✅ 元素查找测试通过")

def test_css_selector():
    """CSS选择器测试"""
    html = """
    <div class="container">
      <ul class="items">
        <li class="item">1</li>
        <li class="item active">2</li>
        <li class="item">3</li>
      </ul>
    </div>
    """
    soup = BeautifulSoup(html, "html.parser")
    items = soup.select("li.item")
    assert len(items) == 3
    active = soup.select_one("li.active")
    assert active.string == "2"
    print("✅ CSS选择器测试通过")

def test_table_extraction():
    """表格提取测试"""
    html = """
    <table>
      <tr><th>名称</th><th>值</th></tr>
      <tr><td>A</td><td>100</td></tr>
      <tr><td>B</td><td>200</td></tr>
    </table>
    """
    soup = BeautifulSoup(html, "html.parser")
    headers = [th.get_text(strip=True) for th in soup.select("th")]
    rows = []
    for tr in soup.select("tr")[1:]:
        cells = [td.get_text(strip=True) for td in tr.select("td"]
        if cells:
            rows.append(dict(zip(headers, cells)))
    assert len(rows) == 2
    assert rows[0]["名称"] == "A"
    print("✅ 表格提取测试通过")

if __name__ == "__main__":
    test_basic_parsing()
    test_find_elements()
    test_css_selector()
    test_table_extraction()
    print("\n🎉 第17课全部验证通过！")
✅ 基本解析测试通过 ✅ 元素查找测试通过 ✅ CSS选择器测试通过 ✅ 表格提取测试通过 🎉 第17课全部验证通过！

🔑 本课要点

  1. CSS 选择器 > find——select() 更灵活，与前端知识通用
  2. get_text(strip=True)——提取文本自动去空白
  3. get("attr", default)——安全获取属性，不存在不报错
  4. 爬虫礼仪——限速、User-Agent、遵守 robots.txt
  5. 缓存策略——避免重复请求，开发调试更友好
← 上一课 📚 目录 下一课 →