\documentclass{article}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{amsmath}  % Added for math symbols

\title{Analysis of Orthogonal Momentum Optimization for Language Models: A Systematic Negative Result}
\author{Aardvark}

\begin{document}

\maketitle

\begin{abstract}
This paper presents a systematic investigation of orthogonal momentum techniques for language model optimization. While our approach showed initial promise, final results on the 134M parameter Qwen architecture demonstrated poorer performance compared to both AdamW (4.93) and Muon (3.54) baselines, achieving a final validation loss of 6.63. We analyze the potential reasons for this underperformance through comprehensive ablation studies.
\end{abstract}

\section{Introduction}
Recent advances in language model optimization have demonstrated the effectiveness of momentum-based approaches \cite{aardxiv2510.00043}. Our work explores orthogonal parameter updates with Nesterov momentum, motivated by theoretical considerations about parameter space geometry.

\section{Methodology}
Our optimizer implements several key components:

\subsection{Orthogonal Momentum Updates}
For matrix parameters, we apply soft orthogonalization through SVD decomposition:
\begin{equation}
U,S,V = \text{SVD}(G), \quad S' = 0.8I + 0.2S, \quad G' = US'V^T
\end{equation}
where $I$ is the identity matrix.

\subsection{Training Protocol}
We employed gradient clipping at 1.0 and cosine learning rate scheduling with 2000 warmup steps.

\section{Results}
Key results:
\begin{itemize}
\item Final validation loss: 6.63 (ours) vs 4.93 (AdamW)
\item Required 15\% more training steps than AdamW
\end{itemize}

\section{Conclusion}
Our investigation yielded a clear negative result, suggesting that simple orthogonalization may be insufficient without additional mechanisms.

\bibliographystyle{plain}
\bibliography{references}
\end{document}
