导论（习题）
1 导论
1.1
\[\nabla_\mathbf{w}E(\mathbf{w})=\sum_{n=1}^N\{y(x_n,\mathbf{w})-t_n\}\phi(x_n)=0
\]
\[\sum_{n=1}^N\phi^\text{T}(x_n)\mathbf{w}\phi(x_n)=\sum_{n=1}^Nt_n\phi(x_n)
\]
\[\mathbf{w}\sum_{n=1}^N\phi(x_n)\phi^\text{T}(x_n) = \sum_{n=1}^Nt_n\phi(x_n)
\]
1.2
1.29
由于\(ln(\cdot)\)是凹函数，利用琴生不等式，
\[\begin{aligned}\text{H}[x]&=-\sum_{i=1}^Mp_i\ln p_i\\&=\sum_{i=1}^Mp_i\ln \frac{1}{p_i}\\&\leq\ln\left(\sum_{i=1}^Mp_i\cdot \frac{1}{p_i}\right)\\&=\ln M.\end{aligned}
\]
1.30
\[\begin{aligned}\text{KL}(p\Vert q)&=-\int p(x)\ln\frac{q(x)}{p(x)}{\rm{d}}x\\&=-\int p(x)\ln\left(\frac{\sigma}{s}\exp\left(-\frac{1}{2}\left[\frac{(x-m)^2}{s^2}-\frac{(x-\mu)^2}{\sigma^2}\right]\right)\right){\rm{d}}x\\&=-\int p(x)\left[\ln\frac{\sigma}{s}-\frac{1}{2}\left(\frac{(x-m)^2}{s^2}-\frac{(x-\mu)^2}{\sigma^2}\right)\right]{\rm{d}}x
\\&=\ln\frac{s}{\sigma}-\frac{1}{2}\int\left[\left(\frac{1}{\sigma^2}-\frac{1}{s^2}\right)x^2+2\left(\frac{m}{s^2}-\frac{\mu}{\sigma^2}\right)x+\left(\frac{\mu^2}{\sigma^2}-\frac{m^2}{s^2}\right)\right]p(x){\rm{d}}x
\\&=\ln\frac{s}{\sigma}+\frac{1}{2}+\frac{\sigma^2+(\mu-m)^2}{2s^2}.
\end{aligned}\]
