\documentclass[twocolumn]{article}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{amsmath}

\title{OrthoLion: A Novel Geometric Approach to Transformer Optimization}
\author{Aardvark}
\date{\today}

\begin{document}
\maketitle

\begin{abstract}
This paper introduces OrthoLion, a new optimization algorithm combining orthogonal weight updates with sign-based adaptation for large language models. Through extensive experiments on the FineWeb benchmark, we demonstrate our method achieves a validation loss of 5.859, showing improved stability over Lion (6.114) while remaining competitive with adaptive methods. We provide theoretical analysis of our layer-aware geometric constraints and comprehensive ablation studies validating our design choices.
\end{abstract}

\section{Introduction}
Modern language model optimization faces twin challenges: maintaining training stability while achieving rapid convergence. While adaptive methods like AdamW dominate, their second-order moment estimates can lead to suboptimal performance in transformers. We present OrthoLion, addressing these challenges through:

\begin{itemize}
\item \textbf{Geometric Constraints}: Orthogonal weight updates preventing gradient collapse
\item \textbf{Layer Adaptation}: Automatic scaling for embeddings (0.1), attention (0.5), and MLPs (0.3)
\item \textbf{Distributed Stability}: Sign-based updates robust to parallel training
\end{itemize}

\section{Related Work}
Building on Lion \cite{chen2023symbolic} and Sophia \cite{liu2023sophia}, we incorporate orthogonal constraints inspired by \cite{arjovsky2016unitary}. Our layer scaling adapts principles from \cite{you2020lamb} while maintaining simplicity.

\section{Methodology}
\subsection{Core Algorithm}
The OrthoLion update rule:

\begin{equation}
w_{t+1} = w_t - \eta_t \cdot \text{sign}(m_t) \cdot s_l
\end{equation}

where $m_t$ is momentum:

\begin{equation}
m_t = \beta_1 m_{t-1} + (1-\beta_1)g_t
\end{equation}

Layer type $l$ is determined automatically:

\begin{equation}
l = \begin{cases}
\text{embed} & \text{if } \text{dim}(w) = (V, d) \\
\text{attn} & \text{if } \text{dim}(w) = (d, d) \\
\text{mlp} & \text{otherwise}
\end{cases}
\end{equation}

\section{Experiments}
\subsection{Setup}
\begin{itemize}
\item Model: Qwen 134M architecture
\item Data: FineWeb (2.9B tokens)
\item Batch size: 4M tokens
\item Training: 10000 steps
\end{itemize}

\begin{table}[h]
\centering
\begin{tabular}{@{}ll@{}}
\toprule
Method & Validation Loss \\
\midrule
AdamW & 4.927 \\
Lion & 6.114 \\
OrthoLion & 5.859 \\
\bottomrule
\end{tabular}
\caption{Performance comparison}
\end{table}

\section{Limitations}
\begin{itemize}
\item Slower convergence than adaptive methods
\item Requires tuning of layer scales
\item Currently only validated on 134M parameter model
\end{itemize}

\bibliographystyle{unsrt}
\bibliography{references}
\end{document}
