% Training MSE of a least-squares line against the sample size M
% Textbook: Part I, "Uni-Variate Linear Least-Squares", exercise
% "simulation: regression error against sample size"
%
% Data from y = w1*x + w0 + eps, Var[eps] = sigma^2, fitted by LS on the same
% data. The training MSE grows with M toward sigma^2: after fitting two
% coefficients, only M-2 of the M errors are free, E[MSE] = sigma^2 (M-2)/M.
% Small samples look better than they are.

clearvars; close all
rng(0)

w1 = 2; w0 = 1;
sigma = 1;
Ms = [4 5 10 20 50 100 200];
R = 30;                         % repetitions per M

mse = zeros(R, length(Ms));
for i = 1:length(Ms)
    M = Ms(i);
    for r = 1:R
        x = rand(M, 1);
        y = w1*x + w0 + sigma*randn(M, 1);
        w = [x ones(M, 1)]\y;               % LS fit
        mse(r, i) = mean((y - [x ones(M, 1)]*w).^2);
    end
end

figure
boxchart(mse)
xticklabels(string(Ms))
hold on
plot(1:length(Ms), sigma^2*(Ms - 2)./Ms, 'r-o', 'LineWidth', 1.5)
yline(sigma^2, 'k--')
hold off
xlabel('M')
ylabel('training MSE')
legend('simulation', '\sigma^2(M-2)/M', '\sigma^2', 'Location', 'southeast')
grid on

fprintf('%5s %10s %10s\n', 'M', 'mean MSE', 'theory')
fprintf('%5d %10.3f %10.3f\n', [Ms; mean(mse); sigma^2*(Ms - 2)./Ms])