\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm,graphicx}
\usepackage{fullpage}
\usepackage[capitalise,nameinlink]{cleveref}
\usepackage{float}
\usepackage{physics}
\crefname{lemma}{Lemma}{Lemmas}
\crefname{fact}{Fact}{Facts}
\crefname{theorem}{Theorem}{Theorems}
\crefname{corollary}{Corollary}{Corollaries}
\crefname{claim}{Claim}{Claims}
\crefname{example}{Example}{Examples}
\crefname{problem}{Problem}{Problems}
\crefname{setting}{Setting}{Settings}
\crefname{definition}{Definition}{Definitions}
\crefname{assumption}{Assumption}{Assumptions}
\crefname{subsection}{Subsection}{Subsections}
\crefname{section}{Section}{Sections}

\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\varepsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}
\newcommand{\trm}[1]{\textrm{#1}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 270: Combinatorial Algorithms and Data Structures
        } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}[section]
\newtheorem*{theorem*}{Theorem}
\newtheorem{itheorem}{Theorem}

\newtheorem{subclaim}{Claim}[theorem]
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem*{proposition*}{Proposition}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem*{lemma*}{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem*{conjecture*}{Conjecture}
\newtheorem{fact}[theorem]{Fact}
\newtheorem*{fact*}{Fact}
\newtheorem{exercise}[theorem]{Exercise}
\newtheorem*{exercise*}{Exercise}
\newtheorem{hypothesis}[theorem]{Hypothesis}
\newtheorem*{hypothesis*}{Hypothesis}
\newtheorem{conjecture}[theorem]{Conjecture}

\theoremstyle{definition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{setting}[theorem]{Setting}
\newtheorem{construction}[theorem]{Construction}
\newtheorem{example}[theorem]{Example}
\newtheorem{question}[theorem]{Question}
\newtheorem{openquestion}[theorem]{Open Question}
\newtheorem{algorithm}[theorem]{Algorithm}
\newtheorem{problem}[theorem]{Problem}
\newtheorem{protocol}[theorem]{Protocol}
\newtheorem{assumption}[theorem]{Assumption}
\newtheorem{exercise-easy}[theorem]{Exercise}
\newtheorem{exercise-med}[theorem]{Exercise}
\newtheorem{exercise-hard}[theorem]{Exercise$^\star$}
\newtheorem{claim}[theorem]{Claim}
\newtheorem*{claim*}{Claim}

\newtheorem{remark}[theorem]{Remark}
\newtheorem*{remark*}{Remark}
\newtheorem{observation}[theorem]{Observation}
\newtheorem*{observation*}{Observation}

% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
% \topmargin 0pt
% \advance \topmargin by -\headheight
% \advance \topmargin by -\headsep
% \textheight 8.9in
% \oddsidemargin 0pt
% \evensidemargin \oddsidemargin
% \marginparwidth 0.5in
% \textwidth 6.5in

\parindent 0in
% \parskip 1.5ex

\begin{document}

\lecture{10 --- February 16, 2023}{Spring 2023}{Prof.\ Jelani Nelson}{Rohit Agarwal}


\section{Recap and Overview}
First, we will wrap up the final result from Fusion trees, namely state how to find the most-significant set bit (MSSB) in constant time.
Then, we will discuss data structures which use hashing.

\section{Most-Significant Set Bit in $O(1)$}
Recall from last time that finding the most-significant bit in constant time is an important operation in implementing Fusion trees in the Word RAM model.
In particular, when we compare the (imperfect) sketch of query with those of elements in a node. At a node  this hinged on thinking about the
indicator on branch bits in a node, $B$, and picking a 
magic $m = \sum_{i = 0}^{r - 1} 2^{m_i}$ with some properties, and defining
\[ \left(\trm{sk}(q) := ((q \& B) \times m) \& \qty(\sum_{i = 0}^{r - 1} 2^{b_i + m_i})\right) \gg (m_0 + b_0) \]
We were guaranteed by the properties of $m$ that this sketch is only $O(r^4)$ bits long.
We saw last time how to find in constant time where $\trm{sk}(node)$ and $\trm{sk}(q)$ agree. We will apply this principle
to see this.

The main idea to take $\trm{MSSB}(x)$ is to break up $x$ into $\sqrt{w}$ blocks, each of size $\sqrt{w}$. Call $F$ to be the
filter with a 1 every $\sqrt{w}$ bits, starting with the MSB. ANDing $x$ and $F$, zeroes out every position except the MSB
of each block. Furthermore, ANDing $x$ and $\overline{F}$, where the bar is bitwise NOT, gives us a mask that keeps the rest of the things.
Subtracting this from $F$ yields a parallel comparison, wherein all nonzero blocks have leading coefficient 0 (since the subtraction had to carry)
and all zero blocks have leading coefficient 1. One more bitwise NOT and mask with $F$ makes it so the MSB of a block is 1 if and only if 
$x$ had a non-MSB that was set. ORing with the original MSB check gives us $z_i$s which indicate whether blocks of $x$ where nonempty:
\[ z = (x \& F) | (\overline{F - (x \& (\overline{F}))} \& F) = \underbrace{z_1 0\dots 0} \underbrace{z_2 0 \dots 0} \dots \underbrace{z_{\sqrt{w}} 0 \dots 0} \]
We use underbraces to denote blocks. Now we know which blocks are nonzero; we must extract the leading one of these. The following claim
is proved in pset 3:
\begin{claim}
  There exist  $0\le m'<u$, $0\le t\le w$, and mask $0\le s < u$ that does `perfect squishing,'
wherein 
\[g = ( \qty(z \cdot m')\gg t) \& s = \underbrace{0 \dots 0}_{w - \sqrt{w}} \underbrace{z_i}_{\sqrt{w}} \]
\end{claim}
\noindent We next do a parallel comparison (e.g. subtraction) $P - Q$ where
\[ P = \underbrace{110\dots0} \underbrace{101\dots0} \dots \underbrace{100\dots1} = \sum_{i=0}^{\sqrt w - 1}(2^{(i+1)(\sqrt w + 1) - 1} + 2^{i + i(\sqrt{w}+1)})\]
and
\[ Q = \underbrace{0 g} \underbrace{0 g} \dots \underbrace{0g} \]
We know $g$ lies between some powers of two, so the comparison will yield a $0$ for the powers that are bigger, then yield $1$ and keep yielding $1$.
We just need to count the number of ones, which we showed how to do last time with a multiplication by $G$, the filter containing blocks
which all have LSB 1 and doing an appropriate shift and mask. Finally, this tells us what the MSB of
$g$ is, i.e. which $z_i$ is 1.
We have now found what block of $x$ the MSSB is in. To finish off the operation, all we need to do is repeat the algorithm one more time on that block,
which will give us our MSSB.

\section{Hashing}
In hashing, we use a random function $h: [u] \to [m]$, called a \textbf{hash function}.

\subsection{Dictionary}
Suppose we are solving the dictionary problem on a universe of size $u$.
Recall hashing with chaining; we initialize $m$ ``bins" and $h(x)$ tells you which bin the item should go in.
If there is a hashing collision, where two items hash to the same thing, then we instead create a linked list with both the items.
To query, you have to walk along the linked list to find your queried item.

\begin{center}
\includegraphics*[width=250px]{chaining.jpeg}
\end{center}

\begin{claim}
  For all $x \in [u]$, the expected time to query $x$ is $O(1 + \frac{n}{m})$.
\end{claim}
In static dictionary, there is a known data structure to take linear space and have constant time query.
However, there is no known algorithm for this regime in the dynamic problem, nor is there a lower bound disallowing it.

However, this approach is not great for cache reasons, so instead we use linear probing.
We still keep an array of size $m$, but when inserting $x$ and finding a collision, we start at $h(x)$
and continue along in the array until we find an empty space. We do a similar walk for a query.

\begin{center}
\includegraphics[width=250px]{lprobing.jpeg}
\end{center}

\noindent Note that if $n$ is close to $m$, then one has to walk for a long time to find an empty space. In fact,
\begin{theorem}[\cite{knuth:1963}]
  In a hash table with linear probing with $m = (1 + \epsilon)n$, then
  \[ \mathbb{E}(\text{query time}) = O(1 / \epsilon^2) \]
\end{theorem}

\subsection{Load Balancing}
We discuss the load balancing problem, which is useful
in the realm of cloud computing. Suppose there are $m$ machines, $n$ jobs
and no machine is too overloaded (ideally each machine gets $\approx \frac{n}{m}$ jobs). There could be a centralized approach
where we know all the jobs and just delegate them evenly. However, it's often in practice faster if there is no centralization, so we will study only
assigning jobs at random. The simplest way to do this is uniformly randomly sending jobs to machines.

Let's take the case where $m = n$. Our criteria for success is that with low probability, the max load is too big.
This is a tail bound, so the Chernoff bound is natural here.

\begin{theorem}[Chernoff bound]
  Let $X_1, \dots, X_n$ be independent random variables supported on $\{0, 1\}$ where $\mathbb{E} X_i = p_i$ and $X = \sum X_i$.
  Then, calling $\mu = \mathbb{E} X = \sum_{i = 1}^n p_i$, for all $\epsilon > 0$:
  \[ \Pr(X > (1 + \epsilon)\mu) < \qty[\frac{e^{\epsilon}}{(1 + \epsilon)^{1 + \epsilon}}]^{\mu} \]
\end{theorem}

\begin{proof}
  We use the Markov bound and the idea of moment-generating functions. Introducing a parameter $t$ and minimizing over it:
  \begin{align*}
    \Pr(X > (1 + \epsilon) \mu ) &= \Pr(\exp(tX) > \exp((1 + \epsilon) \mu t)) \\
                                 &< \frac{\E{\exp(tX)}}{e^{(1 + \epsilon) \mu t}} &&\text{(Markov's Inequality)}\\
    &= \frac{\E\left[\prod_i \exp(t X_i)\right]}{e^{(1 + \epsilon) \mu t}}\\
    &=  \frac{\prod_{i = 1}^n {\E{\exp(tX_i)}}}{e^{(1 + \epsilon) \mu t}} &&\text{(independence of the }X_i\text{)}\\
    &=   \frac{\prod_{i = 1}^n \qty(p_i e^t + (1 - p_i))}{e^{(1 + \epsilon) \mu t}} \\
    &=  \frac{\prod_{i = 1}^n \qty(1 + p_i(e^t - 1))}{e^{(1 + \epsilon) \mu t}} \\
    &\leq  \frac{\prod_{i = 1}^n \exp(p_i(e^t - 1))}{e^{(1 + \epsilon) \mu t}} &&\text{(}1+z\le \exp(z)\text{)}\\
    &=  \frac{\exp(\mu(e^t - 1))}{e^{(1 + \epsilon) \mu t}} \\
    &=  \exp(\mu (e^t - 1 - t - \epsilon t))
  \end{align*}
  Taking the log and differentiating the right hand side for minimization reveals:
  \[ e^{t^*} - 1 - \epsilon = 0 \implies t^* = \ln \qty(1 + \epsilon) \]
  Picking this value of $t$ then yields:
  \[ \Pr(X > (1 + \epsilon) \mu ) < \exp(\mu (1 + \epsilon - 1 - (1 + \epsilon)\ln(1 + \epsilon))) = \qty[\frac{e^{\epsilon}}{(1 + \epsilon)^{1 + \epsilon}}]^{\mu} \]
\end{proof}

\noindent There are two regimes to keep in mind when trying to understand the RHS of the Chernoff bound.
%\begin{corollary}
%  We consider the size of $\epsilon$, since we can choose it.
  \begin{itemize}
  \item If $\epsilon \gg 1$ (perhaps, say 2), then $(1 + \epsilon) \approx \epsilon$ and one should consider $e^{\epsilon}$ as small
  \[ \Pr(X > (1 + \epsilon)\mu) \lessapprox \epsilon^{-\epsilon \mu} \] 
  \item If $\epsilon \ll 1$, we recall that by Taylor expansion $\ln(1+\epsilon) = \epsilon - \epsilon^2/2 + O(\epsilon^3)$. Then 
  \begin{align*}
    \Pr(X > (1 + \epsilon)\mu) &< \frac{e^{\epsilon}}{(1 + \epsilon)^{1 + \epsilon}} \\
    &= \exp(\mu(\epsilon - (1 + \epsilon) \ln(1 + \epsilon)))\\
    &\approx \exp(\mu\qty(\epsilon - (1 + \epsilon) \qty(\epsilon - \frac{\epsilon^2}{2}))) \\
    &\approx \exp(-\epsilon^2 \mu /2)
  \end{align*}
  \end{itemize}
%\end{corollary}

We also will argue the following, which will be a useful result for load balancing.
\begin{lemma}
  If $k^k = n^p$ with $p \geq 1$, then $k = \frac{c \log n}{\log \log n}$ for some $c$ depending on $p$.
\end{lemma}

\begin{proof}
  First let's find an input to $f(x) = x^x$ that undershoots the target $n$:
  \begin{align*}
    \frac{k}{c} \log \frac{k}{c} &= \frac{\log n}{\log \log n} \cdot \log\qty(\frac{\log n}{\log \log n}) \\
    &= \frac{\log n}{\log \log n} \cdot \qty(\log \log n - \log \log \log n) \\
    &=\log n\qty(1 - \frac{\log \log \log n}{\log \log n}) \\
    &< p \log n
  \end{align*}

  And furthermore, pick some $K$ and follow the same logic, yielding:
  \begin{align*}
    \frac{Kk}{c} \log \frac{Kk}{c} &= K \log n\qty(1 - \frac{\log \log \log n - \log K}{\log \log n}) \\
    &>  K \log n\qty(1 - \frac{\log \log \log n}{\log \log n}) \\
    &> p \log n
  \end{align*}
  for a sufficient constant $K$, due to the growth rates of the functions involved.

  This means that there exists some constant in between where we have equality. Call $c$ this constant.
\end{proof}

Now to use this for load balancing, pick a machine and define $X_i$ as the indicator that job $i$ maps to a machine. The load on that machine is thus $X = \sum_{i = 1}^n X_i$.
In expectation it is 1, since $m = n$.
Pick $p = 10$ and thus $k := 1 + \epsilon = \frac{c \log n}{\log \log n}$ for some $c$ we will fix later. Then this is much greater than 1, and the bound tells us
\[ \Pr\qty(X > \frac{c \log n}{\log \log n}) \lesssim \frac{1}{k^k} < \frac{1}{n^{10}} \]
Thus, we have vanishingly low probability of too much load for any fixed machine. Then by a union bound
$$
\Pr(\exists\text{ machine with load } > \frac{c\log n}{\log\log n}) < n\cdot \frac 1{n^{10}} = \frac 1{n^9}
$$
\begin{thebibliography}{1}

\bibitem{knuth:1963}
Donald Knuth.
\newblock Notes on ``open" addressing, 1963.
\newblock URL: http://jeffe.cs.illinois.edu/teaching/ datastructures/2011/notes/knuth-OALP.pdf.

\end{thebibliography}


\end{document}