\documentclass[11pt]{article}
\usepackage{amsmath,amssymb,amsthm}
\usepackage{fullpage}
\usepackage[capitalise,nameinlink]{cleveref}
\usepackage{algorithm,graphicx}
\usepackage[noend]{algpseudocode}
\crefname{lemma}{Lemma}{Lemmas}
\crefname{fact}{Fact}{Facts}
\crefname{theorem}{Theorem}{Theorems}
\crefname{corollary}{Corollary}{Corollaries}
\crefname{claim}{Claim}{Claims}
\crefname{example}{Example}{Examples}
\crefname{problem}{Problem}{Problems}
\crefname{setting}{Setting}{Settings}
\crefname{definition}{Definition}{Definitions}
\crefname{assumption}{Assumption}{Assumptions}
\crefname{subsection}{Subsection}{Subsections}
\crefname{section}{Section}{Sections}
\usepackage{cite}
\DeclareMathOperator*{\E}{\mathbb{E}}
\let\Pr\relax
\DeclareMathOperator*{\Pr}{\mathbb{P}}

\newcommand{\eps}{\varepsilon}
\newcommand{\inprod}[1]{\left\langle #1 \right\rangle}
\newcommand{\R}{\mathbb{R}}

\newcommand{\handout}[5]{
  \noindent
  \begin{center}
  \framebox{
    \vbox{
      \hbox to 5.78in { {\bf CS 270: Combinatorial Algorithms and Data Structures
        } \hfill #2 }
      \vspace{4mm}
      \hbox to 5.78in { {\Large \hfill #5  \hfill} }
      \vspace{2mm}
      \hbox to 5.78in { {\em #3 \hfill #4} }
    }
  }
  \end{center}
  \vspace*{4mm}
}

\newcommand{\lecture}[4]{\handout{#1}{#2}{#3}{Scribe: #4}{Lecture #1}}

\newtheorem{theorem}{Theorem}[section]
\newtheorem*{theorem*}{Theorem}
\newtheorem{itheorem}{Theorem}

\newtheorem{subclaim}{Claim}[theorem]
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem*{proposition*}{Proposition}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem*{lemma*}{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem*{conjecture*}{Conjecture}
\newtheorem{fact}[theorem]{Fact}
\newtheorem*{fact*}{Fact}
\newtheorem{exercise}[theorem]{Exercise}
\newtheorem*{exercise*}{Exercise}
\newtheorem{hypothesis}[theorem]{Hypothesis}
\newtheorem*{hypothesis*}{Hypothesis}
\newtheorem{conjecture}[theorem]{Conjecture}

\theoremstyle{definition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{setting}[theorem]{Setting}
\newtheorem{construction}[theorem]{Construction}
\newtheorem{example}[theorem]{Example}
\newtheorem{question}[theorem]{Question}
\newtheorem{openquestion}[theorem]{Open Question}
% \newtheorem{algorithm}[theorem]{Algorithm}
\newtheorem{problem}[theorem]{Problem}
\newtheorem{protocol}[theorem]{Protocol}
\newtheorem{assumption}[theorem]{Assumption}
\newtheorem{exercise-easy}[theorem]{Exercise}
\newtheorem{exercise-med}[theorem]{Exercise}
\newtheorem{exercise-hard}[theorem]{Exercise$^\star$}
\newtheorem{claim}[theorem]{Claim}
\newtheorem*{claim*}{Claim}

\newtheorem{remark}[theorem]{Remark}
\newtheorem*{remark*}{Remark}
\newtheorem{observation}[theorem]{Observation}
\newtheorem*{observation*}{Observation}

\usepackage{filecontents}

\begin{filecontents}{general.bib}
@inproceedings{van1975preserving,
  title={Preserving order in a forest in less than logarithmic time},
  author={van Emde Boas, Peter},
  booktitle={16th Annual Symposium on Foundations of Computer Science (sfcs 1975)},
  pages={75--84},
  year={1975},
  organization={IEEE}
}
@article{willard1983log,
  title={Log-logarithmic worst-case range queries are possible in space $\Theta$ (N)},
  author={Willard, Dan E},
  journal={Information Processing Letters},
  volume={17},
  number={2},
  pages={81--84},
  year={1983},
  publisher={Elsevier}
}

@inproceedings{fredman1990blasting,
  title={Blasting through the information theoretic barrier with fusion trees},
  author={Fredman, Michael L and Willard, Dan E},
  booktitle={Proceedings of the twenty-second annual ACM symposium on Theory of Computing},
  pages={1--7},
  year={1990}
}

@inproceedings{raman1996priority,
  title={Priority queues: Small, monotone and trans-dichotomous},
  author={Raman, Rajeev},
  booktitle={Algorithms—ESA'96: Fourth Annual European Symposium Barcelona, Spain, September 25--27, 1996 Proceedings 4},
  pages={121--137},
  year={1996},
  organization={Springer}
}

@inproceedings{han2002deterministic,
  title={Deterministic sorting in O (n log log n) time and linear space},
  author={Han, Yijie},
  booktitle={Proceedings of the thiry-fourth annual ACM symposium on Theory of computing},
  pages={602--608},
  year={2002}
}

@inproceedings{han2002integer,
  title={Integer sorting in O (n/spl radic/(log log n)) expected time and linear space},
  author={Han, Yijie and Thorup, Mikkel},
  booktitle={The 43rd Annual IEEE Symposium on Foundations of Computer Science, 2002. Proceedings.},
  pages={135--144},
  year={2002},
  organization={IEEE}
}

@inproceedings{puatracscu2006time,
  title={Time-space trade-offs for predecessor search},
  author={P{\u{a}}tra{\c{s}}cu, Mihai and Thorup, Mikkel},
  booktitle={Proceedings of the thirty-eighth annual ACM symposium on Theory of computing},
  pages={232--240},
  year={2006}
}

@inproceedings{patrascu2007planning,
  title={Planning for fast connectivity updates},
  author={Patrascu, Mihai and Thorup, Mikkel},
  booktitle={48th Annual IEEE Symposium on Foundations of Computer Science (FOCS'07)},
  pages={263--271},
  year={2007},
  organization={IEEE}
}
\end{filecontents}




% 1-inch margins, from fullpage.sty by H.Partl, Version 2, Dec. 15, 1988.
% \topmargin 0pt
% \advance \topmargin by -\headheight
% \advance \topmargin by -\headsep
% \textheight 8.9in
% \oddsidemargin 0pt
% \evensidemargin \oddsidemargin
% \marginparwidth 0.5in
% \textwidth 6.5in

% \parindent 0in
% \parskip 1.

\newcommand{\ZZ}{\mathbb{Z}}
\newcommand{\RR}{\mathbb{R}}
\newcommand{\CC}{\mathbb{C}}
\newcommand{\QQ}{\mathbb{Q}}
\renewcommand{\AA}{\mathbb{A}}
\newcommand{\NN}{\mathbb{N}}
\newcommand{\FF}{\mathbb{F}}
\newcommand{\DD}{\mathbb{D}}
\newcommand{\HH}{\mathbb{H}}
\newcommand{\PP}{\mathbb{P}}
\newcommand{\BB}{\mathbb{B}}
\newcommand{\EE}{\mathbb{E}}

\renewcommand{\exp}{\textup{exp}}
\newcommand{\dist}{\textup{dist}}
\newcommand{\Out}{\textup{Out}}
\newcommand{\In}{\textup{In}}
\newcommand{\key}{\textsf{key}}
\newcommand{\val}{\textsf{val}}
\newcommand{\pred}{\textsf{pred}}
\renewcommand{\insert}{\textsf{insert}}
\newcommand{\delete}{\textsf{delete}}
\newcommand{\vEB}{\textsf{vEB}}
\newcommand{\cluster}{\textsf{cluster}}
\newcommand{\summary}{\textsf{summary}}
\newcommand{\prev}{\textsf{prev}}
\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\begin{document}


\lecture{8 --- February 9, 2023}{Spring 2023}{Prof.\ Jelani Nelson}{Kishan Jani, Wilson Wu}

\section{Overview}

The next few weeks we will be talking about data structures. Today and Tuesday will be about Word RAM model. We want to talk about the predecessor problem. 

\section{The Predecessor Problem}
The predecessor problem has the following parameters and objectives: 
\begin{itemize}
    \item We are given a database of $(\key,\val)$ pairs, with keys ranging in $\{0,1,2,\ldots,u-1\}$. 
    \item The goal is to efficiently (both in time and space) compute $\pred(x),$ which returns $\max\{z\in S: z<x\}$, where $S$ is the set of keys. 
    \item There are two variants of the problem: \emph{static} and \emph{dynamic}. For the static case, we are given all the keys from the start and no changes to the keys are allowed. For the dynamic case, we are also allowed to define $\insert(x)$ and $\delete(x)$ queries, doing exactly what the name suggests.
    \item For the static case, given the array of keys, simply sort the keys. Then for each query $\pred(x)$, binary search for the predecessor of $x$. This takes time $O(n\log n)$ for sorting and $O(\log n)$ for each query $\pred(x)$ using binary search, where $n$ is the size of our database. We can achieve similar runtime bounds for dynamic case, where we use a binary search tree instead of a sorted array. For example, Red-Black trees and AVL trees give $O(\log n)$ time for all three queries. 
\end{itemize}
How do you break ties: what if two items have the same key, and for $\pred(x),$ either item works? There are ways to resolve this, but we will not worry about this issue right now. The second thing to note here is that the above data structures and algorithms rely solely on comparison and sorting of keys. However, this is not the most reasonable case: typically, we are allowed many operations in $O(1)$ time. As we will see, exploiting these operations can lead to faster algorithms. 

\section{Word RAM Model}

Approximate moral: Constant time operations in C (the programming language) are constant time in the Word RAM Model. Following are the assumptions of the model: 
\begin{enumerate}
    \item $u=2^w$ is the largest supportable integer under bitwise operations. We take keys as integers between $0$ and $u-1$. 
    \item Our CPU has some built-in operations: $\text{LOAD},\text{STORE},+,-,\times,/,\textsf{bitshift},$ etc. These operate on ``words" of $w$ bits in O(1) time.  
\end{enumerate}
These assumptions end up leading to significant improvements in the Predecessor problem:
\begin{itemize}
    \item \cite{van1975preserving} gave the van Emde Boas (vEB) tree, which achives $O(\log \log u)$ time per query and takes space $O(u)$. This can, however, be improved to $O(n)$ time, where $n$ is the size of our input. 
    \item \cite{willard1983log} gets the same bounds using $y$-fast tries. 
    \item \cite{fredman1990blasting} developed fusion trees (and came up with the coolest paper name ever), which achieve $O(\log_w n)$ queries for the static case. 
    \item \cite{raman1996priority} came up with dynamic fusion trees to deal with the dynamic case of the problem, also in $O(\log_w n)$. 
\end{itemize}

\begin{proposition}
If you have a good dynamic solution to $\textup{\pred}$ problem, you have a good sorting algorithm.
\end{proposition}
\begin{proof} How do we use dynamic $\pred$ to sort? Given as input the array to be sorted, we insert keys into the data structure one by one and keep track of the $\max$. Then starting at the max, we keep calling $\pred(\cdot)$ to get elements in descending order. There are a total of $n$ insertions and $n-1$ calls to the $\pred$ query, giving a runtime of $O\left( n\left[ T_{\insert}(n) + T_{\pred}(n)\right] \right)$.
\end{proof}
Using this approach, vEB trees imply $O(n \log w)$ time sorting algorithm, and Fusion trees imply $O(n \frac{\log n}{\log w})$ time sorting. Thus you can get $O(n \min \{ \log w, \frac{\log n}{\log w}\})$ by using the faster algorithm based on $n$ and $w$. The worst case is when both are equally fast, which occurs for $\log w = \sqrt{\log n}$, yielding a runtime of $O(n\sqrt{\log n})$. 

Assuming the word RAM model, other fast sorting algorithms not using predecessor data structures are also possible.
\begin{itemize}
    \item \cite{han2002deterministic} showed that you can sort in $O(n\log \log n)$ time deterministically. 
    \item \cite{han2002integer} sorted in $O(n\sqrt{\log \log n})$ with a randomized algorithm. 
    \item Sorting in $O(n)$ time under Word RAM is still an open problem.
\end{itemize}
Unfortunately, the runtime of predecessor data structures is lower bounded.
\begin{theorem}[\cite{puatracscu2006time},\cite{patrascu2007planning}]
    For predecessor, we cannot do better than $\min \{ T_{\vEB}, T_{\textsf{fusion}} \}$. 
\end{theorem}
Now let us begin taking a closer look at these improvements under the word RAM model. 

\section{vEB Trees}
\subsection{Motivation}
 B-trees are a common data structure used in databases. Compared to binary trees, they have a general branching factor of $B$ rather than $2$. The depth then is $\log_B n$ instead of $\log n$. Although they achieve the same asymptotic runtime as binary search trees, they run a lot faster empirically. The reason is that it fits better on disks. For non-flash drive disks, there is a spinning magnetic disk. To read something, you have to wait until the disk spins to where the reading portion is. If this takes a lot of time, in an amortized sense, it is a good idea to wait and read some adjacent data instead. B-trees optimize for this latency by balancing the extra time it takes to read adjacent data with the time it takes the disk to spin to the desired page. By taking into account this disc access model, B-trees are able to perform much better than binary search trees. Similarly, by taking into account the word RAM model, vEB trees allow for faster queries.

 \subsection{Construction}
 The basic idea here is divide and conquer. These are parametrized by the size of the universe, which is initially $u$ (with $0\le \key \le u-1$). They are recursively defined. Fields stored by the node $\vEB_u$: 
\begin{itemize}
    \item $\max$, which is the largest $\key \in S$. 
    \item $\min$, which is the smallest $\key \in S$. 
    \item $\cluster[0,1,\ldots, \sqrt{u} -1]$, an array of pointers to $\sqrt u$ total $\vEB_{\sqrt u}$ trees. 
    \item $\summary$, pointer to a $\vEB_{\sqrt{u}}$ which stores the indices of the non-empty trees in the cluster 
\end{itemize}
\begin{figure}[h]
    \centering
    \includegraphics[scale=0.15]{vEB tree.jpg}
    \caption{$\vEB$ tree}
\end{figure}
Now, we outline the process: 
\begin{itemize}
    \item[1.] Treat $x\in \{0,\ldots, u-1\}$ (now denoted via $[N]=\{0,1,\ldots n-1\}$) as $x= \langle c,i\rangle$, with both $c,i\in [\sqrt{u}]$. That is, we split $x$ into the left $c$ and right $i$ halves of its bit representation.
    \item[2.] If $x$ is in the database, we store $i$ recursively in $V.\cluster[c]$.  
    \item[3.] $c$ is inserted into $V.\summary$ if $S$ contains any key that starts with $c$. Keep all trees empty when you start. 
    \item[4.] If you have only one element in a tree, store it in $\min$ or $\max$ field.
\end{itemize}
\subsection{Predecessor}
\begin{algorithm}[H]
\begin{algorithmic}[1]
    \caption{$\pred(x)$}
    \Require $x$, which is a key from our database.  
    \Ensure $\pred(x)=\max\{z\in S: z<x\}$
    \State Find the decomposition $x=\langle c,i\rangle$ in $O(1)$ time (under the Word RAM model, extractable in constant time by reading two halves via bit-shifts). 
    \If{$x>V.\max$} 
    \State \Return $V.\max$
    \EndIf
    \If{$V.\summary$ contains $c$}
    \If{$i>V.\cluster[c].\min$} 
    \State \Return $\langle c, V.\cluster[c].\pred(i)\rangle $ \:\: \textit{The predecessor is in cluster[c]}
     \Else 
     \State $c'\gets\pred(V.\summary.(c))$       \:\:                   \textit{Find which cluster c', that the predecessor is in}
     \State \Return $\langle c', V.\cluster[c'].\max\rangle $ \:\:
     \EndIf
     \EndIf
\end{algorithmic}
\end{algorithm}

\textbf{Runtime analysis:} There is a recursive step in line $6 + 8$ on $\vEB_{\sqrt u}$, while rest the operations are constant time. Hence the recurrence for runtime is 
\[T(u) = T(\sqrt u) + O(1) , \text{ thus } T(w)=T(w/2) + O(1).\]
By the master theorem, we have $T=O(\log w) = O(\log \log u)$. Notice that the coefficient of $T(\sqrt u)$ is $1$ because the recursive steps of lines $6$ and $8$ happen under disjoint conditions. \\
\subsection{Insert}
\begin{algorithm}[H]
\begin{algorithmic}[1]
    \caption{$\insert(V,x=\langle c,i\rangle)$}
    \Require $x=\langle c, i\rangle$, which is a key from our database, and a cluster.
    \Ensure $x$ is inserted into the cluster. 
    \If{$V.\min = \text{null}$}
    \State $V.\min, V.\max \gets x$, \Return  \:\: \textit{base case: empty tree}
    \EndIf
    \If{$x<V.\min$}
    \State $\text{swap}(x,V.\min)$ \:\: \textit{set V.min to x and insert the original V.min into a child tree}
    \EndIf
    \If{$x>V.\min$}
    \State $V.\max \gets x$
    \EndIf
    \If{$V.\cluster[c]=\text{null} $}   
    \State $V.\summary.\insert(c)$. 
    \EndIf
    \State $\insert(V.\cluster[c],i)$
\end{algorithmic}
\end{algorithm}

\textbf{Runtime analysis:} I have to do two recursive $\insert$s, hence 
\[T(u) = 2 T(\sqrt{u}) + O(1), \;\: \: T(w) \le 2T(w/2) + O(1),\]
which is time $O(w)$; this is bad. However, this is sloppy, we can are only ever actually making one recursive $\insert$ call through the if statements, hence it should be the same recurrence (thus same runtime) as before: since $T(u) = T(\sqrt u) + O(1)$, we have $T(w) = O(\log \log w)$. What is the space complexity? We have
\[S(u) \le (\sqrt u +1 ) S(\sqrt u) + O(\sqrt u)\]
for the $\sqrt u +1$ clusters to deal with recursively. Furthermore, $O(\sqrt u)$ is for all the cluster $\min$ and $\max$ values. Replacing $u$ with $w$ and setting $S(u)= S(2^{w/2})=S'(w)$, we get 
\[S'(w) \le 2^{w/2} S'(w/2) + 2^{w/2},\]
the branching factors go by $2^{w/2},2^{w/4},$ and so on. The time then ends up being $O(2^w) = O(u)$. This is bad news. \\
We want to improve this, which is accomplished by storing $\cluster$ as a hash table instead of as an array. When we say hash table, we mean a solution to the dynamic dictionary problem. We have randomized solutions to hashing with linear space and expected constant time operations. We will talk about other non-170 ways to make good dictionaries later, but let's just accept it and move on for now. We have 
\[V.\cluster = \text{hashtable mapping $c$ to a pointer to child tree}\]
Space is now proportional to number of clusters rather than $u$. Memory proportional to sum of sizes of non-empty VEB trees throughout recursion
At each level, since you either recurse into the summary or recurse into the cluster, the time is based on depth which yields $O(n\log w)$ memory.

\section{$x$-fast tries}
Here is a really simple way to solve $\pred$. The database say is $\{0,2,3,7,8,9,12,13\}$. Here $w=4$ and $u=16$. Let us store this as a bit array: 
\[1011000111001100\]
Build a perfect binary tree where these bits are the leaves, and each internal node is the logical $\texttt{OR}$ of two children bits. See figure 2.
\begin{figure}[h]
    \centering
    \includegraphics[scale=0.2]{bin tree.jpg}
    \caption{$x$-fast trie}
\end{figure}

What is $\pred$ of an element? Store the $1$ positions in a $w$-linked list so that we can jump between $w$ in $O(1)$ time. If someone asks what the predecessor of $6$ is, we walk up the tree; we were its left child, which means if I go for the smallest in the right sub-tree, this gives the successor and jumping back to the previous $1$ gives me the predecessor via my linked list. However, this process takes $O(w)$ time, based on tree depth.\\
Notice though the path to the root is monotone due to us using $OR$, so we can binary search on the ancestry of $p$. 
\begin{itemize}
    \item Store tree in a length $2u-1$ array as for a binary heap.
    \item For a node indexed at $x$ (indexed from $1$), the left child is at $2x$ while the right child is at $2x+1$. Thus in order to find parent, all I have to do is $\lfloor \cdot/2 \rfloor$, which is a bit-shift. Thus to find the $k$th ancestor, all I need is $k$ bit-shifts, which takes time $O(k)$ under Word RAM. This gives $O(\log w) = O(\log \log u)$ time. 
\end{itemize}
Queries are fast while inserts are slow. $y$-fast tries, that we will see next time, make fast insertion over linear space. 

\bibliography{general}{}
\bibliographystyle{plain}
\end{document}
