Cross-validation

Source: CrossValidation/crossvalidation.Rmd

The page contrasts in-sample and out-of-sample prediction using lapylace fits for the regression models and scikit-learn for fold bookkeeping.

Setup

from pathlib import Path
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import lapylace as lp

root = Path("../../ROS-Examples")

def coef_medians(fit):
    return pd.Series(
        [np.median(fit.alpha_draws()), *np.median(fit.beta_draws(), axis=0)],
        index=["Intercept", *fit.columns],
    )

def fit_glm(formula, data, family=None, seed=1, prior_scale=2.5, intercept_scale=5, aux_scale=10, **kwargs):
    return lp.stan_glm(
        formula,
        data=data,
        family=family or lp.gaussian(),
        prior=lp.normal(0, prior_scale),
        prior_intercept=lp.normal(0, intercept_scale),
        prior_aux=lp.exponential(aux_scale),
        chains=2,
        parallel_chains=2,
        iter_warmup=300,
        iter_sampling=500,
        seed=seed,
        refresh=100,
        **kwargs,
    )

from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error

kidiq = pd.read_csv(root / "KidIQ/data/kidiq.csv")
kidiq.head()

	kid_score	mom_hs	mom_iq	mom_work	mom_age
0	65	1	121.117529	4	27
1	98	1	89.361882	4	25
2	85	1	115.443165	4	27
3	83	1	99.449639	3	25
4	115	1	92.745710	4	27

formulas = ["kid_score ~ mom_hs", "kid_score ~ mom_hs + mom_iq", "kid_score ~ mom_hs * mom_iq"]
kf = KFold(n_splits=5, shuffle=True, random_state=19201)
rows = []
for j, formula in enumerate(formulas):
    rmses = []
    for fold, (tr, te) in enumerate(kf.split(kidiq)):
        fit = fit_glm(formula, kidiq.iloc[tr], seed=19210 + 10*j + fold, prior_scale=10, intercept_scale=30, aux_scale=30)
        pred = fit.posterior_epred(kidiq.iloc[te]).mean(axis=0)
        rmses.append(np.sqrt(mean_squared_error(kidiq.iloc[te].kid_score, pred)))
    rows.append({"formula": formula, "cv_rmse": np.mean(rmses)})
pd.DataFrame(rows)

	formula	cv_rmse
0	kid_score ~ mom_hs	19.830143
1	kid_score ~ mom_hs + mom_iq	18.245287
2	kid_score ~ mom_hs * mom_iq	18.185056