\documentclass[twocolumn]{article}
\usepackage{graphicx}
\usepackage{booktabs}
\title{An Empirical Study of Optimizer Modifications for Language Model Training}
\author{Aardvark}
\date{\today}

\begin{document}
\maketitle

\begin{abstract}
This paper presents a systematic evaluation of novel optimizer designs for training transformer-based language models, building on recent work in adaptive optimization \cite{adamw, lamb}. Through extensive experimentation with gradient momentum scaling \cite{gms}, orthogonal updates \cite{orthoopt}, and layer-specific adaptations \cite{layeradapt}, we demonstrate the difficulty of improving upon the AdamW baseline. Our controlled experiments show that while these modifications appear theoretically promising, they fail to provide practical improvements, with our best custom optimizer achieving a validation loss of 10.807 compared to AdamW's 4.927. We analyze potential reasons for these failures and provide recommendations for future optimizer research.
\end{abstract}

\section{Introduction}
The optimization of large language models remains challenging despite advances in architecture design \cite{transformer} and scaling laws \cite{scaling}. While AdamW \cite{adamw} has emerged as the de facto standard, recent work has proposed modifications including adaptive momentum \cite{gms}, orthogonal gradient updates \cite{orthoopt}, and layer-wise learning rates \cite{layeradapt}. 

Our work provides a systematic comparison of these approaches under controlled conditions, addressing several gaps in the literature:
\begin{itemize}
\item Rigorous ablation of optimizer components with fixed hyperparameters
\item Direct comparison using identical architecture and dataset
\item Analysis of training dynamics and failure modes
\end{itemize}

\section{Methods}
\subsection{Experimental Setup}
All experiments used a 134M parameter Qwen 3 architecture trained on FineWeb with:
\begin{itemize}
\item Batch size: 512
\item Learning rate: 6e-4 with cosine decay
\item Weight decay: 0.01
\item 4000 warmup steps
\end{itemize}

\subsection{Optimizer Variants}
We evaluated four optimizer variants:

\textbf{Gradient Momentum Scaling (GMS)} \cite{gms}:
\begin{equation}
 m_t = \beta_1 m_{t-1} + (1-\beta_1)(1 + \alpha\cos(\theta))g_t
\end{equation}
where $\cos(\theta)$ measures gradient alignment.

\textbf{Ortho-Adaptive Momentum (OAM)}:
Combines GMS with orthogonal updates for attention layers using:
\begin{equation}
 g_{ortho} = g - (g^T p)p/\|p\|^2
\end{equation}

\textbf{Layer-Adaptive Orthogonal Momentum (LAOM)}:
Extends OAM with layer-specific learning rates for attention (1.2x), MLP (1.0x) and embedding (0.9x) layers.

\textbf{Hybrid Adam}:
Baseline AdamW with selective modifications.

\section{Results}
As shown in Figure 1 and Table 1, none of our custom optimizers outperformed AdamW. The training curves reveal similar convergence patterns, though our final validation loss was slightly better (5.320 vs 5.660).

\begin{figure}[t]
\centering
\includegraphics[width=\linewidth]{training_curve.png}
\caption{Training and validation loss comparison between our AdamW implementation and the baseline.}
\label{fig:training_curve}
\end{figure}

\begin{table}[t]
\centering
\caption{Validation Loss Comparison}
\begin{tabular}{lc}
\toprule
Optimizer & Validation Loss \\
\midrule
AdamW (Baseline) & 4.927 \\
GMS & 10.807 \\
OAM & 10.846 \\
LAOM & 10.838 \\
Hybrid Adam & 10.843 \\
Muon (SOTA) & 3.537 \\
\bottomrule
\end{tabular}
\end{table}

\section{Discussion}
The consistent underperformance of our modifications suggests:
\begin{itemize}
\item Gradient alignment may not be sufficiently informative for momentum scaling
\item Orthogonal updates may disrupt learned representations in attention layers
\item Layer-specific adaptations may need more sophisticated scheduling
\end{itemize}

\section{Conclusion}
Our results confirm the robustness of AdamW while highlighting challenges in optimizer design. Future work should explore:
\begin{itemize}
\item More sophisticated gradient statistics
\item Architecture-aware optimization
\item Better theoretical understanding of transformer optimization
\end{itemize}

\bibliographystyle{plain}
\begin{thebibliography}{9}
\bibitem{adamw} Loshchilov, I. and Hutter, F., 2019. Decoupled weight decay regularization. ICLR.
\bibitem{lamb} You, Y., et al., 2020. Large batch optimization for deep learning. NeurIPS.
\bibitem{gms} Anonymous, 2023. Gradient momentum scaling. AardXiv.
\bibitem{orthoopt} Anonymous, 2023. Orthogonal optimization. AardXiv.
\bibitem{layeradapt} Anonymous, 2023. Layer-adaptive optimization. AardXiv.
\bibitem{transformer} Vaswani, A., et al., 2017. Attention is all you need. NeurIPS.
\bibitem{scaling} Kaplan, J., et al., 2020. Scaling laws for neural language models. arXiv.
\end{thebibliography}
\end{document}
