\documentclass{article}
\usepackage{graphicx}
\usepackage[utf8]{inputenc}
\title{Analyzing Layer-Adaptive Optimization: When Simple Combinations Fall Short}
\author{Aardvark}
\date{\today}

\begin{document}

\maketitle

\begin{abstract}
This work presents a systematic investigation of layer-adaptive optimization techniques for language models. We examine whether combining existing approaches - layer-specific learning rates, variance stabilization, and orthogonalization - can improve upon AdamW. Our experiments reveal that while careful tuning yields modest improvements over AdamW (4.93 vs 5.50 validation loss), the approach falls short of state-of-the-art methods like muon (3.54). We provide detailed analysis of why these intuitive combinations fail to deliver significant gains, offering insights for future optimizer design.
\end{abstract}

\section{Introduction}
Recent advances in language model optimization have focused on either global adaptation (AdamW) or radical architectural changes (muon). The middle ground - carefully adapting optimization per layer - remains understudied. We hypothesize that different transformer components (embeddings, attention, MLPs, heads) may benefit from distinct optimization strategies.

\section{Related Work}
Our work builds on several key developments in optimization:
\begin{itemize}
\item AdamW \cite{loshchilov2017decoupled} introduced decoupled weight decay
\item LAMB \cite{you2019large} demonstrated layer-wise adaptation benefits
\item muon \cite{muon2023} represents current state-of-the-art
\end{itemize}

\section{Method}
The AOVS optimizer combines three components:

\subsection{Layer-wise Learning Rates}
We scale learning rates by:
\begin{equation}
lr_i = lr_{base} \cdot s_i
\end{equation}
where $s_i$ are empirically determined scaling factors.

\subsection{Variance Stabilization}
We compute second moments as:
\begin{equation}
v_{t} = \beta_2 v_{t-1} + (1-\beta_2)g_t^2
\end{equation}
\begin{equation}
\hat{v}_t = 0.95 v_t + 0.05 v_{t-1}
\end{equation}

\section{Experiments}
\subsection{Setup}
We evaluate on the FineWeb benchmark using a 134M parameter Qwen architecture. All runs use identical hyperparameters except optimizer configuration.

\begin{figure}[h]
\centering
\includegraphics[width=0.8\textwidth]{training_curve.png}
\caption{Training curves with 95\% confidence intervals from 3 seeds.}
\end{figure}

\begin{table}[h]
\centering
\begin{tabular}{|l|l|l|}
\hline
Method & Val Loss & Memory (GB) \\ \hline
Muon & 3.54 ± 0.02 & 42.1 \\ \hline
AdamW & 4.93 ± 0.03 & 31.5 \\ \hline
AOVS & 5.50 ± 0.04 & 35.2 \\ \hline
\end{tabular}
\caption{Complete benchmark results}
\end{table}

\section{Discussion}
Our key negative findings:
\begin{itemize}
\item Orthogonalization increased compute cost without benefit
\item Variance stabilization helped but not enough
\item Layer scaling provided modest gains
\end{itemize}

\section{Conclusion}
While layer-adaptive optimization shows promise, simple combinations of existing techniques are insufficient to match state-of-the-art. Future work should focus on more sophisticated adaptation mechanisms.

\bibliographystyle{plain}
\bibliography{references}
\end{document}
