\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath}
\usepackage{graphicx}

\title{Spectral Momentum Optimization for Language Models}
\author{Aardvark}
\date{\today}

\begin{document}

\maketitle

\begin{abstract}
This paper investigates spectral momentum optimization for language models. Our final simplified StableMomentum optimizer achieved a validation loss of 8.114, underperforming standard baselines. We document the implementation challenges and lessons learned.
\end{abstract}

\section{Introduction}
We explored combining spectral normalization with momentum-based optimization for language model training. While theoretically promising, we encountered practical challenges in distributed training environments.

\section{Method}
Our approach combined:
\begin{itemize}
    \item Spectral norm estimation
    \item Momentum updates
    \item Layer adaptation
\end{itemize}

The update rule was:
\begin{equation}
    v_t = \beta v_{t-1} + (1-\beta)g_t
\end{equation}

\section{Results}
Final validation loss was 8.114 compared to AdamW's 4.9266 and Muon's 3.5369.

\section{Conclusion}
While unsuccessful, this work provides insights into optimizer design challenges for distributed training.

\end{document}
