\documentclass[twocolumn]{article}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{amsmath}

\title{StableLion: Robust Sign-Based Optimization Through Layerwise Adaptation}
\author{Aardvark}

\begin{document}

\maketitle

\begin{abstract}
We present StableLion, a stabilized sign-based optimizer combining layerwise learning rate adaptation with gradient norm clipping for language model pretraining. While sign-based methods like Lion offer memory efficiency, they often suffer from training instability. StableLion addresses this through three mechanisms: (1) parameter-specific trust ratios bounding update magnitudes, (2) layerwise learning rate adaptation inspired by LAMB, and (3) gradient norm stabilization. On a 134M parameter Qwen model trained on FineWeb, StableLion achieves 4.931 validation loss, outperforming Lion (6.114) and approaching AdamW (4.927) while using 30\% less memory than adaptive methods. We provide ablation studies and implementation details to support these findings.
\end{abstract}

\section{Introduction}
Recent advances in language model optimization have focused on adaptive methods like AdamW~\cite{loshchilov2017decoupled} and second-order approaches~\cite{liu2023sophia}. Sign-based optimizers~\cite{chen2023symbolic} offer memory efficiency but face stability challenges. Our work bridges this gap by augmenting sign-based updates with careful normalization.

Building on layerwise adaptation techniques from LAMB~\cite{you2019lamb} and trust region methods~\cite{you2017scaling}, StableLion provides:
\begin{itemize}
\item Per-parameter update clipping via gradient/parameter norm ratios
\item Layerwise learning rate scaling
\item Gradient norm stabilization
\end{itemize}

\section{Methodology}
StableLion's update rule combines these elements:

\begin{equation}
\Delta_t = -\eta_l \cdot \text{min}\left(\frac{||\theta_l||}{||g_l||}, \tau\right) \cdot \text{sign}(m_t)
\end{equation}

Where $\eta_l$ is the base learning rate for layer $l$, $\tau$ is the trust ratio maximum (3.0), and $m_t$ is the momentum.

\section{Experiments}
\begin{figure}[t]
\centering
\includegraphics[width=\linewidth]{figures/training_curves.pdf}
\caption{Training curves showing StableLion's stability advantages.}
\label{fig:curves}
\end{figure}

\section{Limitations}
While promising, StableLion has several limitations:
\begin{itemize}
\item Evaluated on single architecture/task
\item Requires careful warmup tuning
\item Small batch performance not verified
\end{itemize}

\bibliographystyle{plain}
\bibliography{references}
\end{document}
