\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{fullpage}
\usepackage[capitalise,nameinlink]{cleveref}
\crefname{lemma}{Lemma}{Lemmas}
\crefname{fact}{Fact}{Facts}
\crefname{theorem}{Theorem}{Theorems}
\crefname{corollary}{Corollary}{Corollaries}
\crefname{claim}{Claim}{Claims}
\crefname{example}{Example}{Examples}
\crefname{problem}{Problem}{Problems}
\crefname{setting}{Setting}{Settings}
\crefname{definition}{Definition}{Definitions}
\crefname{assumption}{Assumption}{Assumptions}
\crefname{subsection}{Subsection}{Subsections}
\crefname{section}{Section}{Sections}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\varepsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 270: Combinatorial Algorithms and Data Structures
        } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}[section]
\newtheorem*{theorem*}{Theorem}
\newtheorem{itheorem}{Theorem}

\newtheorem{subclaim}{Claim}[theorem]
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem*{proposition*}{Proposition}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem*{lemma*}{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem*{conjecture*}{Conjecture}
\newtheorem{fact}[theorem]{Fact}
\newtheorem*{fact*}{Fact}
\newtheorem{exercise}[theorem]{Exercise}
\newtheorem*{exercise*}{Exercise}
\newtheorem{hypothesis}[theorem]{Hypothesis}
\newtheorem*{hypothesis*}{Hypothesis}
\newtheorem{conjecture}[theorem]{Conjecture}

\theoremstyle{definition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{setting}[theorem]{Setting}
\newtheorem{construction}[theorem]{Construction}
\newtheorem{example}[theorem]{Example}
\newtheorem{question}[theorem]{Question}
\newtheorem{openquestion}[theorem]{Open Question}
% \newtheorem{algorithm}[theorem]{Algorithm}
\newtheorem{problem}[theorem]{Problem}
\newtheorem{protocol}[theorem]{Protocol}
\newtheorem{assumption}[theorem]{Assumption}
\newtheorem{exercise-easy}[theorem]{Exercise}
\newtheorem{exercise-med}[theorem]{Exercise}
\newtheorem{exercise-hard}[theorem]{Exercise$^\star$}
\newtheorem{claim}[theorem]{Claim}
\newtheorem*{claim*}{Claim}

\newtheorem{remark}[theorem]{Remark}
\newtheorem*{remark*}{Remark}
\newtheorem{observation}[theorem]{Observation}
\newtheorem*{observation*}{Observation}
\newtheorem{idea}[theorem]{Idea}
\newtheorem*{idea*}{Idea}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
% \topmargin 0pt
% \advance \topmargin by -\headheight
% \advance \topmargin by -\headsep
% \textheight 8.9in
% \oddsidemargin 0pt
% \evensidemargin \oddsidemargin
% \marginparwidth 0.5in
% \textwidth 6.5in

% \parindent 0in
% \parskip 1.5ex

\begin{document}

\lecture{19 --- March 21, 2023}{Spring 2023}{Prof.\ Jelani Nelson}{Calvin Yan, Young Jin Park}

\section{Overview}

In the last lecture we expanded upon primal/dual analysis for approximation algorithms, introduced the integrality gap as an obstacle to approximation, and proposed a solution in the form of polynomial time approximation schemes (PTAS) and fully polynomial time approximation schemes (FPTAS).

In this lecture we will continue analysis of PTAS and FPTAS, specifically with regards to knapsack, and conclude approximation algorithms with the techniques of fully polynomial-time randomized approximation schemes (FPRAS) and SDP-rounding.

\section{Last Time: Greedy 2-Approximation For Knapsack}
Last lecture we claimed without proof that a greedy solution provides a 2-approximation of integral knapsack. This lecture, we will prove that claim.

First, a recap of the algorithm. Intuitively, we can sort each item by by $\frac{v_i}{w_i}$, which represents the cost-effectiveness for some item $i$, then take all the elements that we can starting by the most cost-effective item.

This is optimal for the fractional case, where we can take a fraction of each element. \emph{However}, it is not optimal for integral knapsack. We can make a very simple optimization:

\begin{itemize}
    \item[(1)] Take as much of each item as we can, in descending order of their effectiveness $\frac{v_i}{w_i}$.
    \item[(2)] Take $v_\text{max}$, the largest element we can afford with our space, and nothing else.
    \item[(3)] Use the strongest result between (1) and (2).
\end{itemize}

\begin{lemma} \label{ineq}
Suppose the greedy algorithm takes items $1, 2, \ldots, k - 1$ but not the $k$-th item from our sorted sequence where $\frac{v_1}{w_1} \geq \frac{v_2}{w_2} \geq \ldots \geq \frac{v_k}{w_k}$. Then the sum $\sum_{i=1}^k v_i > \mathsf{OPT}(ILP)$.
\end{lemma}
\begin{proof}
Recall that we can write the LP
\begin{align*}
    \max & \quad \sum_{i = 1}^n x_iv_i\\
    s. t. & \quad \sum_{i = 1}^n x_iw_i \leq W\\
    & \forall i; \; 0 \leq x_i \leq 1
\end{align*}

LP relaxation tells us that
\begin{align*}
    \mathsf{OPT} = \mathsf{OPT}(ILP) \leq \mathsf{OPT}(LP)
\end{align*}

Since fractional knapsack took all of the first $k - 1$ items and some fraction (maybe $0$) of the $k$-th item, we know that the remaining items it took contribute less value than if it took the entirety of the $k$-th item (since it has more weight and contributes more value per weight by our sorting order). We have the strict inequality
\begin{align*}
    \sum_{i=1}^k v_i &> \mathsf{OPT}(LP) \\
    \mathsf{OPT}(ILP) \leq \mathsf{OPT}(LP) &< \sum_{i=1}^k v_i
\end{align*}
\end{proof}

\begin{claim} \label{2approx}
The better of (1) or (2) achieves $\geq \frac{1}{2}\mathsf{OPT}$.
\end{claim}

\begin{proof}

By \cref{ineq},

\begin{align*}
    \sum_{i = 1}^k v_i = \underbrace{\sum_{i = 1}^{k - 1}v_i}_{\leq \text{greedy}} + \underbrace{v_k}_{\leq v_\text{max}} > \mathsf{OPT}
\end{align*}

It then follows that at least one of greedy or $v_\text{max}$ is $> \frac{\mathsf{OPT}}{2}$.
\end{proof}



\section{PTAS for Knapsack}
As a reminder, a polynomial-time approximation scheme aims to find a result $\geq (1 - \epsilon)\mathsf{OPT}$ in time $n^{f(1 / \epsilon)}.

\begin{observation*}
The number of items of value $> \epsilon \mathsf{OPT}$ that the optimal solution takes is at most $\lfloor \frac{1}{\epsilon} \rfloor$.
\end{observation*}


\begin{idea*}
Guess $S = \{\text{$1/\epsilon$ largest items that \textsf{OPT} takes}\}$.
\end{idea*}

For any particular guess $S$, we want to pack the remaining items well. To do this, we can use the greedy approach on the remaining elements, i.e. the elements smaller than $\epsilon \textsf{OPT}$.

However, we don't know the value of $\epsilon \textsf{OPT}$. How can we compare against $\epsilon \textsf{OPT}$ if we don't even know its value? We can guess the lowest value of any item in $S$ and set $\tau := \epsilon \textsf{OPT}$ slightly lower than this, greedily packing the items with less than $\tau$ value. There are $O(n)$ such thresholds, so we can just try them all. An extra $O(n)$ factor of runtime is fine, since it's still polynomial in $n$.

How much do we lose from using greedy in this remaining portion? Recall from the proof for \cref{2approx} that $\textsf{Greedy} > \textsf{OPT} - v_k$, so we lose at most $v_k$ which is $\leq v_{\text{max}} \leq \epsilon \mathsf{OPT}$ by construction. This gives us the value $(1-\epsilon)\textsf{OPT}$, which is exactly what we want.\\

The final runtime is equal to

\begin{align*}
    (\text{number of guesses for } S) \times \underbrace{(\text{time per } S)}_{\text{poly}(n)}
\end{align*}

The number of guesses for $S$ is the number of sets of $\leq \frac{1}{\epsilon}$ items we can take. A neat combinatorical trick for computing this is to add $\frac{1}{\epsilon}$ dummy items and compute the equivalent quantity

\begin{align*}
    \underbrace{n + \frac{1}{\epsilon} \choose \frac{1}{\epsilon}}_{\substack{
            \text{Upperbounded by number of} \\
            \text{subsets of } \{1, \ldots, n\} \text{ of size } \leq \frac{1}{\epsilon}
    }}
    \leq (e(\epsilon n + 1))^{\frac{1}{\epsilon}}
\end{align*}

Note the exponential dependence on $\frac{1}{\epsilon}$, which makes this PTAS, not FPTAS.

\section{FPTAS for Knapsack}

Recall that a fully polynomial time approximation scheme is a PTAS with worst-case poly$(\frac{n}{\epsilon})$ runtime. Results of $\tilde{O}(n + \frac{1}{\epsilon^4})^{[1]}$ and $\tilde{O}(n + \frac{1}{\epsilon^{2.2}})^{[2]}$ have been found, and it remains an open problem to get this bound down to $\tilde{O}(n + \frac{1}{\epsilon^2})$, but these are beyond the scope of today's lecture. Instead, we will demonstrate a relatively simpler approach that achieves $O(\frac{n^3}{\epsilon})$ runtime.

\paragraph{Starting point:} We know an exact DP algo with $O(nV)$ runtime, where $V := \sum_{i=1}^n v_i$

\begin{construction}
Construct a new knapsack problem with values $v_i'$ and same weight $w_i$ s.t. $v_i' := \lfloor \frac{n}{\epsilon} \cdot \frac{v_i}{v_\text{max}} \rfloor$, then run the exact DP solution on the modified problem with $(v_i', w_i)$, and return this optimal set. The runtime of this approximation is 
$O(nV') = O(n \frac{n^2}{\epsilon}) = O(\frac{n^3}{\epsilon})$
\end{construction}



\paragraph{Analysis for FPTAS:}\\

Let $A$ be the optimal set for $v_i$ and $B$ the optimal set for $v_i'$. Observe that for $\alpha = \frac{n}{\epsilon v_\text{max}}$,
\begin{align*}
    & v_i' = \lfloor \alpha v_i \rfloor \\
    & \implies \alpha v_i - 1 \leq v_i' \leq \alpha v_i \\
    & \implies \frac{1}{\alpha} v_i' \leq v_i \leq \frac{1}{\alpha} (v_i' + 1)
\end{align*}
it follows that
\begin{align*}
    \textsf{val}(B) \geq \frac{1}{\alpha} \textsf{val}(B) &\geq \frac{1}{\alpha} \textsf{val'}(A) \\
    &\geq \frac{1}{\alpha} (\alpha \textsf{val'}(A) - |A|)\\
    &= \underbrace{\textsf{val}(A)}_{\mathsf{OPT}} - \frac{\epsilon v_\text{max}}{n} |A| \\
    &\geq \mathsf{OPT} - \epsilon v_\text{max} \\
    &\geq (1 - \epsilon)\mathsf{OPT}
\end{align*}

\section{FPRAS for DNF-counting}

\begin{definition}
A condition is in \underline{conjunctive normal form} (CNF) if it is an AND of a bunch of ORs: $(x_1 \vee \overline{x_3} \vee x_7 \vee x_2) \wedge (\overline{x_3}) \wedge ( \ldots$
\end{definition}

\begin{definition}
A condition is in \underline{disjunctive normal form} (DNF) if it is an OR of a bunch of ANDs: $(x_1 \wedge \overline{x_3} \wedge x_7 \wedge x_2) \vee (\overline{x_3}) \vee ( \ldots$
\end{definition}

\begin{remark*}
CNF satisfiability is merely the SAT problem which we know to be NP-complete. DNF is not (since we can just find an assignment that satisfies one clause!). However, we do have a harder problem regarding DNF.
\end{remark*}


\paragraph{Goal:} Given DNF formula $\phi$, count the number of $x\text{ s.t. }\phi(x)$ is true.

\begin{claim}
    DNF-counting is NP-hard (equivalently, DNF is $\#P$-complete).
\end{claim}

\begin{proof}
    We aim to show that SAT reduces to DNF-counting.\\
    Given a problem in SAT form,
    \begin{align*}
        \phi = \underbrace{(x_1 \vee \overline{x_3} \vee \ldots )}_{C_1} \wedge \underbrace{(\overline{x_7} \vee \ldots )}_{C_2} \wedge \ldots \wedge \underbrace{(\ldots )}_{C_m}
    \end{align*}
    Then 
    \begin{align*}
        \overline{\phi} &= \overline{(x_1 \vee \overline{x_3} \vee \ldots ) \wedge (\overline{x_7} \vee \ldots ) \wedge \ldots \wedge (\ldots )} \\
        &= \overline{(x_1 \vee \overline{x_3} \vee \ldots)} \vee \overline{(\overline{x_7} \vee \ldots )} \vee  \ldots \vee \overline{(\ldots )} \\
        &= (\overline{x_1} \wedge x_3 \wedge \ldots) \vee (x_7 \wedge \ldots ) \vee (\ldots )
    \end{align*}
    which is in disjunctive normal form. If $\overline{\phi}$ has $T$ solutions, then $\phi$ has $2^n - T$ solutions, since the statements are complementary and there are $2^n$ variable assignments in total. Then $\phi$ is satisfiable if and only if DNF-counting finds $T < 2^n$.
\end{proof}
    \\

    \paragraph{New goal:} Find $\tilde{T}$ such that $\Pr(|T - \tilde{T}| > \epsilon T) < \delta$. In more intuitive terms: $\tilde{T} = (1 \pm \epsilon)T$ with high probability.

    \paragraph{Idea 1:} Define $p := \frac{T}{2^n}$. We want to know $\tilde{p} = (1 \pm \epsilon) p$ and return $2^n \tilde{p}$. However, there is a problem with this idea: A small $p$ results in high estimation variance.

    \paragraph{Example:} Consider $\phi = x_1 \wedge x_2 \wedge \ldots \wedge x_n$

    There is only one assignment that satisfies this. When the number of solutions is exponentially small, we will probably not sample any in polynomial time! There is a serious possibility that you will estimate $p = 0$ and return $2^n \times 0 = 0$.

    Thankfully, there is a way to reexpress the estimation that ensures $p$ isn't too small$^{[3]}$:

    \begin{align*}
        B &:= \text{ the set of satisfying assignments }\\
        B' &:= \{(i, x) : x \text{ satisfies } C_i\}\\
        S_i &:= \{x : x \text{ satisfies } C_i\} \\
        \implies B &= \bigcup S_i \\
        B' &= \bigsqcup S_i 
    \end{align*}
    Here $\sqcup$ refers to the disjoint union.

    \paragraph{New goal:} estimate $p' := \frac{|B|}{|B'|}$

    \begin{observation*}
        $|B'| \leq m |B| \implies p' \geq \frac{1}{m}$ \\
        This is because each $x \in B$ appears in B exactly once, but may appear in $B'$ up to $m$ times (for $i = 1, 2, \ldots, m$).
    \end{observation*}

    \paragraph{Algorithm}
    The estimation of $p'$ can be done using Monte Carlo sampling.
    \begin{itemize}
        \item for $j = 1$ to $l$,
        \begin{itemize}
            \item pick random $(i, x) \in B'$
            \item check if $(i, x) \in B$
        \end{itemize}
        \item return $\frac{\text{\# of samples in } B}{l}$
    \end{itemize}
    

    \begin{observation*}
        $B$ is in bijective correspondence with a subset of $B'$.
        \begin{align*}
            B \leftrightarrow \{(i, x) : C_i \text{ is the first clause that $x$ satisfies}\}
        \end{align*}
    \end{observation*}
     How do we sample an element of $B'$ uniformly at random?\\

     To sample a clause at random, pick $i$ with probability $\frac{|S_i|}{\sum_{j = 1}^m |S_j|}$. To sample an assignment at random, assign deterministically the variables in $C_i$ such that it is satisfied, and all other variables uniformly at random.

     \paragraph{Analysis:} Define $X_1, \ldots, X_l$ as follows:
     \begin{align*}
         X_j &:= \begin{cases}
             1 & \text{ if $j$th sample $\in B$}\\
             0 & \text{ otherwise }
         \end{cases} \\
         X &:= \sum_{j=1}^l X_j,
     \end{align*}
     \begin{observation*}
         $\E(X) = p' \cdot l$ due to linearity of expectation.
     \end{observation*}
         
     Then apply the Chernoff bound:
     \begin{align*}
         \Pr(|X - \mu| > \epsilon \mu ) &\leq 2e^{- \epsilon^2 \mu / 3}\\
         &= 2e^{-\epsilon ^2 p'l / 3} \\
         &\leq 2e^{-\epsilon ^2 l / 3m} \leq \delta \\
         l &= \lceil \frac{3m}{\epsilon^2} \ln (\frac{2}{\delta}) \rceil
     \end{align*}

\begin{thebibliography}{42}

\bibitem{LAW79}
Eugene L. Lawler.
\newblock Fast Approximation Algorithms for Knapsack Problems.
\newblock \emph{Math. Oper. Res.}, 4(4): 339-356, 1979.

\bibitem{DXM23}
Mingyang Deng, Ce Jin, Xiao Mao.
\newblock  Approximating Knapsack and Partition via Dense Subset Sums.
\newblock \emph{ Symposium on Discrete Algorithms}, 2961-2979, 2023.

\bibitem{KLM89}
Richard M. Karp, Michael Luby, Neal Madras.
\newblock Monte-Carlo Approximation Algorithms for Enumeration Problems.
\newblock \emph{J. Algorithms}, 10(3):429-448, 1989.

\end{thebibliography}

\end{document}