\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{fullpage}
\usepackage[capitalise,nameinlink]{cleveref}
\crefname{lemma}{Lemma}{Lemmas}
\crefname{fact}{Fact}{Facts}
\crefname{theorem}{Theorem}{Theorems}
\crefname{corollary}{Corollary}{Corollaries}
\crefname{claim}{Claim}{Claims}
\crefname{example}{Example}{Examples}
\crefname{problem}{Problem}{Problems}
\crefname{setting}{Setting}{Settings}
\crefname{definition}{Definition}{Definitions}
\crefname{assumption}{Assumption}{Assumptions}
\crefname{subsection}{Subsection}{Subsections}
\crefname{section}{Section}{Sections}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\varepsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 270: Combinatorial Algorithms and Data Structures
        } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}[section]
\newtheorem*{theorem*}{Theorem}
\newtheorem{itheorem}{Theorem}

\newtheorem{subclaim}{Claim}[theorem]
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem*{proposition*}{Proposition}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem*{lemma*}{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem*{conjecture*}{Conjecture}
\newtheorem{fact}[theorem]{Fact}
\newtheorem*{fact*}{Fact}
\newtheorem{exercise}[theorem]{Exercise}
\newtheorem*{exercise*}{Exercise}
\newtheorem{hypothesis}[theorem]{Hypothesis}
\newtheorem*{hypothesis*}{Hypothesis}
\newtheorem{conjecture}[theorem]{Conjecture}

\theoremstyle{definition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{setting}[theorem]{Setting}
\newtheorem{construction}[theorem]{Construction}
\newtheorem{example}[theorem]{Example}
\newtheorem{question}[theorem]{Question}
\newtheorem{openquestion}[theorem]{Open Question}
% \newtheorem{algorithm}[theorem]{Algorithm}
\newtheorem{problem}[theorem]{Problem}
\newtheorem{protocol}[theorem]{Protocol}
\newtheorem{assumption}[theorem]{Assumption}
\newtheorem{exercise-easy}[theorem]{Exercise}
\newtheorem{exercise-med}[theorem]{Exercise}
\newtheorem{exercise-hard}[theorem]{Exercise$^\star$}
\newtheorem{claim}[theorem]{Claim}
\newtheorem*{claim*}{Claim}

\newtheorem{remark}[theorem]{Remark}
\newtheorem*{remark*}{Remark}
\newtheorem{observation}[theorem]{Observation}
\newtheorem*{observation*}{Observation}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
% \topmargin 0pt
% \advance \topmargin by -\headheight
% \advance \topmargin by -\headsep
% \textheight 8.9in
% \oddsidemargin 0pt
% \evensidemargin \oddsidemargin
% \marginparwidth 0.5in
% \textwidth 6.5in

% \parindent 0in
% \parskip 1.5ex

\usepackage[parfill]{parskip}
\usepackage{graphicx}
\usepackage{dsfont}

\begin{document}

\lecture{LEC 23 --- 2023-04-11}{Spring 2023}{Prof.\ Jelani Nelson}{Johannes Pfannschmidt}

\section{Overview}

In the last lecture we covered strong duality, complementary slackness, and the ellipsoid algorithm.

This lecture's topic is the interior point method (IPM) for linear programming. 
This includes a look at newton's method, with a detour to gradient descent.

\section{IPM}
\subsection{Input} The interior point method takes as input an LP of the form:
\begin{align*}
	\min_{x} c^T x \\
	\text{s.t. } Ax\geq b
\end{align*}

\subsection{Basic Idea}
The idea is to define a series of functions:
\begin{align*}
	\lambda\in [0, \infty )\\
	f_\lambda(x):=\lambda c^T x + p(s(x))\\
	\text{where } p \text{ is s.t. } p(z)\rightarrow\infty\\
	\text{if any } z_i\rightarrow 0
\end{align*}
$s(x):=Ax-b$ is called the slack-vector, and $p$ is called the barrier function, as it keeps the point interior to the LP by penalization.

For this lecture, $p(s(x)):=-\sum_{i=1}^{m}ln(s(x)_i)$.

\begin{figure}[h]
	\centering
	\includegraphics[width=0.4\textwidth]{polytope.jpg}
	\caption{Rough visualization of the IPM method}
	\label{fig:polytope}
\end{figure}

For IPM, we start at the optimal $x$ for $\lambda=0$, and then gradually increase $\lambda$ while continuously adapting $x$ to stay optimal for the current $\lambda$.
In doing so, we move from the polytope's analytical center to the optimal vertex along the unique central path, as visualized in Fig. \ref{fig:polytope}.

Since the algorithm is actually iterative and moves in discrete steps, we in reality don't move exactly along the central path, but stay "close" to it.
We differentiate levels of "closeness":

\begin{itemize}
	\item \textbf{central}: $x$ is on central path (and gradient is $0$)
	\item \textbf{awesome}: Norm of gradient is is tiny $(\leq \frac{1}{100})$
	\item \textbf{ok}: Norm of gradient is small $(\leq \frac{1}{3})$
\end{itemize}

\subsection{Overall architecture}
The algorithm operates as follows:
\begin{enumerate}
	\item  set $\lambda_0$ to be very small: $\lambda_0 = \exp(-cL)$
	\item get $\tilde{x}(\lambda_0)$ that is awesome
	\item for $i=1..k$\newline
	\vspace{-0.5cm}
	\begin{itemize}
		\item $\lambda_i=(1+\gamma)\lambda_{i-1}$
		\item run $O(1)$ Newton iterations on $\tilde{x}(\lambda_{i-1})$ for $f_{\lambda_{i}}$ to obtain $\tilde{x}(\lambda_{i})$ that is awesome
	\end{itemize}
\end{enumerate}
\vspace{0.5cm}
($L$ is the precision of the problem.)

For this to work, $\gamma$ needs to be sufficiently small such that if $\tilde{x}(\lambda_{i-1})$ is awesome, $x(\lambda_{i})$ is still at least ok.
Values $\gamma=O(\frac{1}{\sqrt{m}})$ and $k=O(\sqrt{m}L)$ will work. 

\subsection{Finding $\tilde{x}(\lambda_0)$ in step 2.}
To find our starting point, we pick an $N$ to be really big ($\exp(cL)$), and formulate a new LP: 
\begin{align*}
	&\min_{x} c^Tx + Nz\\
	\text{s.t. }\quad & Ax+z \cdot\mathds{1} \geq b\\
	& 0 \leq z \leq 2^{L+1}\\
	& -2^{L+1}\mathds{1} \leq x \leq 2^{L+1}\mathds{1}	
\end{align*}

This LP has an easy interior point: $z=||b||_0,x=\vec{0}$.
\vspace{0.5cm}

Using
\begin{align}
	\nabla f_\lambda(x)&=\lambda c - A^TS^{-1}_x\mathds{1}\\
	\nabla^2 f_\lambda(x)&= A^TS^{-2}_x A
\end{align}
and $||x||_A=\sqrt{x^TAx}$ for PSD $x$,
we define "centrality" for $\lambda$ as $\delta_\lambda(x):=||\nabla f_\lambda(x)||_{(\nabla^2f_\lambda(x))^{-1}}$.

This is also the norm we use for above mentioned definition of "awesome" ($\delta_\lambda(x)\leq\frac{1}{100}$) and "ok" ($\delta_\lambda(x)\leq\frac{1}{3}$).


To now find our $\tilde{x}(\lambda_0)$ for step 2., we first set $\lambda = 1$, and introduce a different cost function $c'=A^TS^{-1}_{(x_0, z_0)}\mathds{1}$, such that the gradient is $1c'-A^TS^{-1}_{(x_0,z_0)}\mathds{1}=0$, giving us a point on the central path.

To obtain a starting point on the central path for our original cost function, we apply step 3. in reverse,  decreasing $\lambda$ from one towards zero.
Since for sufficiently small $\lambda$ the cost function does not influence the gradient much, we can then switch back to the old cost function, and our point is still "awesomely close" to the central path. 
We have found our $\tilde{x}(\lambda_0)$.

\subsection{3rd step}
The question remains of how to decide how large $k$ in the 3rd step should be:

\begin{align*}
	\text{let } x(\lambda)&:=\min_{x} f_\lambda(x)\\
	0 &= <0, x(\lambda)-x^*>\\
	&= <\lambda c -A^TS^{-1}_x(\lambda)\mathds{1}, x(\lambda)-x^*>\\
	&\Rightarrow \lambda c^T (x(\lambda)-x^*)=\mathds{1}^TS^{-1}_x(\lambda)A_{(x(\lambda)-x^*)}\\
	&=\mathds{1}^TS^{-1}_{x(\lambda)}(s(x(\lambda)-x^*))\\
	&=\sum_{i=1}^{m}\frac{s(x(\lambda))_i-s(x^*)_i}{s(x(\lambda))_i}\\
	&\leq  =\sum_{i=1}^{m}\frac{s(x(\lambda))_i}{s(x(\lambda))_i} = m\\
	&\Rightarrow c^T(x(\lambda)-x^*)\leq \frac{m}{\lambda}
\end{align*}
where $x^*$ is the perfect (central) $x$ for $\lambda$. 
\vspace{1cm}

If the error $c^T(x(\lambda)-x^*)$ should be at most $\epsilon$, we therefore need $\lambda$ to be $\frac{m}{\epsilon}$.

This is the termination criterion for the 3rd step, we stop iterating if $\lambda \geq \frac{m}{\epsilon}$.
Actually, if $\frac{m}{\lambda}< \exp(-cL)$, we can round to the optimal vertex (proof omitted).
Therefore we are done when $\frac{m}{\lambda} < exp(-cL)$ $\Rightarrow$ $\lambda > m \cdot \exp(cL)$.

Specifically, to be done we need $(1+\gamma)^k\lambda_0 > m\cdot\exp(cL)$ to hold, or derived therefrom $(1+\gamma)^k > m\cdot\exp(2cL)$, which for the number of iterations implies $k\geq \frac{1}{\gamma}\cdot\ln(m\cdot\exp(2cL))=O(\sqrt{m}(L+\lg m))$

\section{Detour into continuous optimization}
\subsection{First-order methods}
\begin{itemize}
	\item given $f: \mathbb{R}^n\rightarrow\mathbb{R}$, $\nabla f$
	\item promised $\forall x: \alpha I \leq \nabla^2 f(x) \leq \beta I$	\footnote{Notation: $A\leq B \Leftrightarrow B-A$ is PSD $\Leftrightarrow \forall z: z^TAz\leq z^TBz$. This is called the Loewner order.}
\end{itemize}

\subsubsection{Basic Idea}
\begin{itemize}
	\item start with some iterate $x_0\in\mathbb{R}^n$
	\item gradually move from $x_k$ to $x_{k+1}$ along negative gradient, such that $f(x_{k+1})<f(x_k)$
\end{itemize}

This is motivated by Taylor's theorem:
\begin{alignat*}{3}
	f(x_{k+1})&=f(x_k) &&+ <\nabla f(x_k), x_{k+1}-x_k>\\
	& &&+ \int_{0}^{1}\int_{0}^{t} <x_{k+1}-x_k, \nabla^2 f(x_\alpha)(x_{k+1}-x_k)>d\alpha\text{ } dt\\
	& &&\text{ where } x_\alpha:=x_k+\alpha(x_{k+1}-x_k) \text{ for }  x\in [0,1]\\ 
	&\leq f(x_k) &&+ <\nabla f(x_k), x_{k+1}-x_k>+\frac{\beta}{2} ||x_{k+1}-x_k||_2^2
\end{alignat*}

Gradient descent then essentially is choosing $x_{k+1}$ to minimize the last line directly above, which gives $x_{k+1}=x_k-\frac{1}{\beta}\nabla f(x_k)$.
With that, said line will be at most $f(x_k)-\frac{1}{2\beta}||\nabla f(x_k)||_2^2$ (not shown).

\subsection{Lemma (proof omitted)}
\[
	\forall k: f(x_{k+1})-f(x^*)\leq (1-\frac{\alpha}{\beta})\cdot (f(x_k)-f(x^*))
\]

$\Rightarrow$ The optimality gap can be halved in $O(\frac{\beta}{\alpha})$ oracle calls.

Notably, a better bound of $O(\sqrt{\frac{\beta}{\alpha}})$ is achievable using "accelerated gradient descent", due to Nesterov \cite{Nesterov83} in '83.
This bound is optimal.

\section{Newton's Method}
\begin{itemize}
	\item goal: $\min f(x)$
	\item given: $f$, $\nabla f$, $\nabla^2 f$
	\item \textbf{Assumption}: 
	\begin{align*}
		&\forall k,\quad x_\alpha:=x_k + \alpha(x_{k+1}-x_k):\\
		&(1-\epsilon)\nabla^2 f(x_k)\leq \nabla^2 f(x_\alpha)\leq (1+\epsilon)\nabla^2 f(x_k)\\
		&\Leftrightarrow\\
		&-\epsilon I \leq A^{\frac{1}{2}}(B-A)A^{\frac{1}{2}}\leq \eps I
	\end{align*}
\end{itemize}

\subsection{Heart of Newton}
In Newton's method, the function $f$ is approximated by its second-order Taylor expansion:

\begin{align*}
	f(x_{k+1})\approx f(x_k) + <\nabla f(x_k), x_{k+1}-x_k> + \frac{1}{2} <x_{k+1}-x_k>, \nabla^2 f(x_k)(x_{k+1}-x_k)>
\end{align*}

$x_{k+1}$ is then choosen to optimize the right-hand side of this equation, which gives:

\[
	x_{k+1}=x_k-(\nabla^2 f(x_k))^{-1} \cdot\nabla f(x_k)
\]

\subsubsection{Newton Progress Lemma}
If $f$ is twice differentiable and \textbf{Assumption} holds, then:

\[
	||\nabla f(x_{k+1})||_{(\nabla^2 f(x_{k+1}))^{-1}} \leq \frac{\epsilon}{1-\epsilon}||\nabla f(x_k)||_{(\nabla^2 f(x_{k}))^{-1}}
\]



\bibliographystyle{alpha}

\begin{thebibliography}{42}

\bibitem{Nesterov83}
Nesterov~Y.,
\newblock A method of solving a convex programming problem with convergence rate $O(1/k^2)$.
\newblock {\em Soviet Mathematics Doklady}, volume 27, pages 137--147, 1983.

\end{thebibliography}

\end{document}