从零到一，完成一个真实的数据分析项目
本项目使用加州1990年人口普查数据，预测各街区的房屋中位数价格。这是一个经典的回归问题，涵盖数据分析全流程的7个步骤。
from sklearn.datasets import fetch_california_housing
import pandas as pd
data = fetch_california_housing()
df = pd.DataFrame(data.data, columns=data.feature_names)
df['MedHouseVal'] = data.target
print(f"数据集大小: {df.shape}") # (20640, 9)
print(f"特征: {list(data.feature_names)}")
# ['MedInc', 'HouseAge', 'AveRooms', 'AveBedrms',
# 'Population', 'AveOccup', 'Latitude', 'Longitude']
print(df.describe().round(2))| 特征 | 含义 | 范围 | 单位 |
|---|---|---|---|
| MedInc | 收入中位数 | 0.5-15.0 | 万美元 |
| HouseAge | 房屋年龄中位数 | 1-52 | 年 |
| AveRooms | 平均房间数 | 0.85-141.9 | 间 |
| AveBedrms | 平均卧室数 | 0.33-34.07 | 间 |
| Population | 人口 | 3-35682 | 人 |
| AveOccup | 平均入住率 | 0.69-1243 | 人/户 |
| Latitude | 纬度 | 32.5-42.0 | 度 |
| Longitude | 经度 | -124~-114 | 度 |
| MedHouseVal | 房价中位数(目标) | 0.15-5.0 | 万美元 |
# 缺失值检查
print(f"缺失值: {df.isnull().sum().sum()}") # 0
# 目标变量统计
print(f"均值: {df['MedHouseVal'].mean():.3f}") # 2.069
print(f"中位数: {df['MedHouseVal'].median():.3f}") # 1.797
print(f"标准差: {df['MedHouseVal'].std():.3f}") # 1.154
# 与目标变量的相关性
corr_with_target = df.corr()['MedHouseVal'].drop('MedHouseVal').sort_values(ascending=False)
for feat, corr in corr_with_target.items():
bar = '█' * int(abs(corr) * 30)
print(f" {feat:>12}: {corr:+.3f} {bar}")
# 关键发现:
# MedInc(+0.688) → 收入是最强预测因子！
# AveRooms(+0.152) → 房间数有正相关
# Latitude(-0.144) → 纬度越高房价越低(北加州较便宜)# 衍生特征（业务直觉驱动）
df['RoomsPerPerson'] = df['AveRooms'] / (df['Population'] / df['HouseAge'].clip(1))
df['BedroomRatio'] = df['AveBedrms'] / df['AveRooms'] # 卧室占比
df['PopDensity'] = df['Population'] / df['AveRooms'].clip(0.1) # 人口密度
print(f"原始特征: 8个 → 增加后: 11个")
print(f"新增: RoomsPerPerson(人均房间), BedroomRatio(卧室占比), PopDensity(人口密度)")from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression, Ridge
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
X = df.drop('MedHouseVal', axis=1)
y = df['MedHouseVal']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
# 标准化
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)
# 模型对比
models = {
'LinearRegression': LinearRegression(),
'Ridge': Ridge(alpha=1.0),
'RandomForest': RandomForestRegressor(n_estimators=100, max_depth=10, random_state=42),
'GradientBoosting': GradientBoostingRegressor(n_estimators=100, max_depth=5, learning_rate=0.1, random_state=42),
}
for name, model in models.items():
model.fit(X_train_scaled, y_train)
y_pred = model.predict(X_test_scaled)
rmse = np.sqrt(mean_squared_error(y_test, y_pred))
r2 = r2_score(y_test, y_pred)
print(f"{name:>20}: RMSE={rmse:.4f}, R²={r2:.4f}")| 模型 | RMSE | MAE | R² | 评价 |
|---|---|---|---|---|
| LinearRegression | 0.7279 | 0.5259 | 0.5956 | 基线 |
| Ridge | 0.7279 | 0.5259 | 0.5956 | ≈基线 |
| RandomForest | 0.5461 | 0.3680 | 0.7724 | 显著提升 |
| GradientBoosting | 0.4914 | 0.3284 | 0.8157 | 🏆最优 |
# 最优模型深入评估
best_model = models['GradientBoosting']
# 交叉验证
from sklearn.model_selection import cross_val_score
cv_scores = cross_val_score(best_model, scaler.transform(X), y, cv=5, scoring='r2')
print(f"5折CV R²: {cv_scores.mean():.4f} ± {cv_scores.std():.4f}")
# 特征重要性
feature_names = list(X.columns)
importances = best_model.feature_importances_
sorted_idx = np.argsort(importances)[::-1]
for i in sorted_idx[:8]:
bar = '█' * int(importances[i] * 50)
print(f" {feature_names[i]:>15}: {importances[i]:.4f} {bar}")
# 预测示例
sample = X_test_scaled[:5]
preds = best_model.predict(sample)
for i in range(5):
err = abs(preds[i] - y_test.iloc[i])
print(f" 样本{i+1}: 实际={y_test.iloc[i]:.3f}, 预测={preds[i]:.3f}, 误差={err:.3f}")# model_deploy.py — Streamlit部署
import streamlit as st
import numpy as np
import pickle
# 加载模型和scaler
with open('model.pkl', 'rb') as f:
model = pickle.load(f)
with open('scaler.pkl', 'rb') as f:
scaler = pickle.load(f)
st.title("🏠 加州房价预测")
# 输入特征
MedInc = st.slider("收入中位数(万$)", 0.5, 15.0, 3.5)
HouseAge = st.slider("房屋年龄(年)", 1, 52, 25)
AveRooms = st.slider("平均房间数", 1.0, 20.0, 5.0)
AveBedrms = st.slider("平均卧室数", 0.1, 5.0, 1.0)
Population = st.slider("人口", 3, 35000, 1500)
AveOccup = st.slider("平均入住率", 0.5, 10.0, 3.0)
Latitude = st.slider("纬度", 32.5, 42.0, 36.0)
Longitude = st.slider("经度", -124.0, -114.0, -119.0)
if st.button("🔮 预测房价"):
features = np.array([[MedInc, HouseAge, AveRooms, AveBedrms,
Population, AveOccup, Latitude, Longitude,
AveRooms/(Population/max(HouseAge,1)),
AveBedrms/AveRooms,
Population/max(AveRooms,0.1)]])
scaled = scaler.transform(features)
pred = model.predict(scaled)[0]
st.success(f"预测房价: ${pred*100000:,.0f}")
# 部署步骤:
# 1. 保存模型: pickle.dump(model, open('model.pkl','wb'))
# 2. 保存scaler: pickle.dump(scaler, open('scaler.pkl','wb'))
# 3. 运行: streamlit run app.py
# 4. 分享: 部署到Streamlit Cloud / Docker