机器学习中的优化 Optimization Chapter 2 Gradient Descent(2)

$\large \bf{Theorem }\ 2.7:$
$f:\mathbb{R^d}\rightarrow\mathbb{R}\text{ be convex and differentiable with a global minimum }x^*;\text{ Suppose }f\text{ is smooth with parameter }L.\text{ Choosing stepsize: }\gamma = \frac{1}{L},\text{ gradients descent yields:}$

\begin{aligned} (1) & f (x_{T}) - f (x^{*}) \leq \frac{L}{2 T} | | x_{0} - x^{*} | |^{2} \end{aligned}

$\begin{align} f(x_T)-f(x^*)\leq \frac{L}{2T}||x_0-x^*||^2 \end{align}$

$\large\bf Proof:$
$f\text{ is differentiable and smooth, according to Lemma 2.6, we can get:}$

\begin{aligned} (2) & f (x_{t + 1}) - f (x_{t}) \leq - \frac{1}{2 L} | | g_{t} | |^{2} \end{aligned}

$\begin{align} f(x_{t+1})-f(x_t)\leq -\frac{1}{2L}||g_t||^2 \end{align}$

$\text{Therefore:}$

\begin{aligned} (3) & \frac{1}{2 L} | | g_{t} | |^{2} \leq f (x_{t}) - f (x_{t + 1}) \end{aligned}

$\begin{align} \frac{1}{2L}||g_t||^2\leq f(x_t)-f(x_{t+1}) \end{align}$

$\text{Now we sum up:}$

\begin{aligned} (4) & \frac{1}{2 L} \sum_{t = 0}^{T - 1} | | g_{t} | |^{2} & \leq \sum_{t = 0}^{T - 1} [f (x_{t}) - f (x_{t + 1})] \\ (5) & = f (x_{0}) - f (x_{T}) \end{aligned}

$\begin{align} \frac{1}{2L}\sum_{t=0}^{T-1}||g_t||^2&\leq \sum_{t=0}^{T-1}[f(x_t)-f(x_{t+1})]\\ &=f(x_0)-f(x_T) \end{align}$

$\gamma = 1/L,\text{ therefore from previous analysis:}$

\begin{aligned} (6) & \sum_{t = 0}^{T - 1} [f (x_{t}) - f (x^{*})] \leq \frac{γ}{2} \sum_{t = 0}^{T - 1} | | g_{t} | |^{2} + \frac{1}{2 γ} | | x_{0} - x^{*} | |^{2} \end{aligned}

$\begin{align} \sum_{t=0}^{T-1}[f(x_t)-f(x^*)]\leq \frac{\gamma}{2}\sum_{t=0}^{T-1}||g_t||^2+\frac{1}{2\gamma}||x_0-x^*||^2 \end{align}$

$\text{Combine (5) and (6):}$

\begin{aligned} (7) & \sum_{t = 0}^{T - 1} [f (x_{t}) - f (x^{*})] & \leq \frac{γ}{2} \sum_{t = 0}^{T - 1} | | g_{t} | |^{2} + \frac{1}{2 γ} | | x_{0} - x^{*} | |^{2} \\ (8) & \leq f (x_{0}) - f (x_{T}) + \frac{1}{2 γ} | | x_{0} - x^{*} | |^{2} \end{aligned}

$\begin{align} \sum_{t=0}^{T-1}[f(x_t)-f(x^*)]&\leq \frac{\gamma}{2}\sum_{t=0}^{T-1}||g_t||^2+\frac{1}{2\gamma}||x_0-x^*||^2 \\ &\leq f(x_0)-f(x_T)+\frac{1}{2\gamma}||x_0-x^*||^2 \end{align}$

$\text{Hence:}$

\begin{aligned} (9) & \sum_{t = 1}^{T} [f (x_{t}) - f (x^{*})] & \leq \frac{1}{2 γ} | | x_{0} - x^{*} | |^{2} \\ (10) & = \frac{L}{2} | | x_{0} - x^{*} | |^{2} \end{aligned}

$\begin{align} \sum_{t=1}^{T}[f(x_t)-f(x^*)]&\leq \frac{1}{2\gamma}||x_0-x^*||^2\\ &=\frac{L}{2}||x_0-x^*||^2 \end{align}$

$\text{As the result:}$

\begin{aligned} (11) & T \cdot (f (x_{T}) - f (x^{*})) & \leq \sum_{t = 1}^{T} [f (x_{t}) - f (x^{*})] \\ (12) & = \frac{L}{2} | | x_{0} - x^{*} | |^{2} \end{aligned}

$\begin{align} T\cdot (f(x_T)-f(x^*))&\leq \sum_{t=1}^T[f(x_t)-f(x^*)]\\ &=\frac{L}{2}||x_0-x^*||^2 \end{align}$

\begin{aligned} (13) & \Rightarrow f (x_{T}) - f (x^{*}) \leq \frac{L}{2 T} | | x_{0} - x^{*} | |^{2} \end{aligned}

$\begin{align} \Rightarrow f(x_T)-f(x^*)\leq \frac{L}{2T}||x_0-x^*||^2 \end{align}$

1. Smooth and strongly convex function: $O(\log(1/\epsilon))$ steps

$\text{First-order method: only use the gradient information to minimize }f.$

$\large\bf Definition\ 2.9:$
$\text{Strongly convex function: }$

\begin{aligned} (14) & f (y) \geq f (x) + \nabla f (x)^{T} (y - x) + \frac{L}{2} | | x - y | |^{2} \end{aligned}

$\begin{align} f(y)\geq f(x)+\nabla f(x)^T(y-x)+\frac{L}{2}||x-y||^2 \end{align}$

$\large \bf Lemma\ 2.10:$
$\text{if }f \text{ is strongly convex with parameter }\mu>0,\text{ then }f\text{ is }\bf{strictly\ convex\ and\ has\ a\ unique\ global\ minimum.}$

$\text{Assume that }f\text{ is stringly convex with }\mu,\text{ from vanilla analysis:}$

\begin{aligned} (15) & g_{t} (x_{t} - x^{*}) & = \nabla f (x_{t})^{T} (x_{t} - x^{*}) \\ (16) & \geq f (x_{t}) - f (x^{*}) + \frac{μ}{2} | | x_{t} - x^{*} | |^{2} \end{aligned}

$\begin{align} g_t(x_t-x^*)&=\nabla f(x_t)^T(x_t-x^*)\\ &\geq f(x_t)-f(x^*)+\frac{\mu}{2}||x_t-x^*||^2 \end{align}$

$\text{Hence:}$

\begin{aligned} (17) & f (x_{t}) - f (x^{*}) & \leq \frac{1}{2 γ} [γ^{2} | | g_{t} | |^{2} + | | x_{t} - x^{*} | |^{2} - | | x_{t + 1} - x^{*} | |^{2}] - \frac{μ}{2} | | x_{t} - x^{*} | |^{2} \end{aligned}

$\begin{align} f(x_t)-f(x^*)&\leq \frac{1}{2\gamma}[\gamma^2||g_t||^2+||x_t-x^*||^2-||x_{t+1}-x^*||^2]-\frac{\mu}{2}||x_t-x^*||^2 \end{align}$

$\text{Rewrite it as:}$

\begin{aligned} (18) & | | x_{t + 1} - x^{*} | |^{2} \leq 2 γ [f (x^{*}) - f (x_{t})] + γ^{2} | | g_{t} | |^{2} + (1 - μ γ) | | x_{t} - x^{*} | |^{2} \end{aligned}

$\begin{align} ||x_{t+1}-x^*||^2\leq 2\gamma [f(x^*)-f(x_t)]+\gamma^2||g_t||^2+(1-\mu\gamma)||x_t-x^*||^2 \end{align}$

$\large\bf{Theorem\ 2.12:}$
$f:\mathbb{R^d}\rightarrow\mathbb{R}\text{ be convex and differnentiable. Suppose }f\text{ is smooth with }L,\text{ and strongly convex with }\mu. \text{ Choosing stepsize:}$

\begin{aligned} (19) & γ = 1 / L \end{aligned}

$\begin{align} \gamma = 1/L \end{align}$

$\text{Gradient descent with arbitary }x_0\text{ satisfies the following two properties:}$
$(i)$

\begin{aligned} (20) & | | x_{t + 1} - x^{*} | |^{2} \leq (1 - \frac{μ}{L}) | | x_{t} - x^{*} | |^{2} \end{aligned}

$\begin{align} ||x_{t+1}-x^*||^2\leq (1-\frac{\mu}{L})||x_t-x^*||^2 \end{align}$

$\large\bf Proof:$
$\text{By smooth, we know:}$

\begin{aligned} (21) & f (x^{*}) - f (x_{t}) \leq f (x_{t + 1}) - f (x_{t}) \leq - \frac{1}{2 L} | | g_{t} | |^{2} \end{aligned}

$\begin{align} f(x^*)-f(x_t)\leq f(x_{t+1})-f(x_t)\leq -\frac{1}{2L}||g_t||^2 \end{align}$

$\text{Combine (18), we get}$

\begin{aligned} (22) & | | x_{t + 1} - x^{*} | |^{2} & \leq - γ^{2} | | g_{t} | |^{2} + γ^{2} | | g_{t} | |^{2} + (1 - μ γ) | | x_{t} - x^{*} | |^{2} \\ (23) & \leq (1 - \frac{μ}{L}) | | x_{t} - x^{*} | |^{2} \end{aligned}

$\begin{align} ||x_{t+1}-x^*||^2&\leq -\gamma^2||g_t||^2+\gamma^2||g_t||^2+(1-\mu\gamma)||x_t-x^*||^2\\ &\leq (1-\frac{\mu}{L})||x_t-x^*||^2 \end{align}$

$(ii)$