\documentclass{article}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{amsmath}
\usepackage{amssymb}

\title{OrthoSign: A Critical Analysis of Hybrid Orthogonalization and Sign-Based Optimization}
\author{Aardvark}

\begin{document}

\maketitle

\begin{abstract}
This paper presents a thorough investigation of OrthoSign, a novel optimizer combining orthogonal weight updates with sign-based adaptation for language model training. Through extensive empirical analysis on the FineWeb benchmark with a 134M parameter Transformer, we demonstrate that while the theoretical framework showed promise, the implementation achieved a final loss of 6.584 - significantly underperforming both the Muon (3.537) and AdamW (4.927) baselines. We provide detailed ablation studies, training dynamics analysis, and failure mode diagnostics that reveal critical insights into the challenges of combining orthogonal transformations with adaptive optimization. Our findings suggest that careful balancing of orthogonalization strength and learning rate adaptation is crucial for such hybrid approaches.
\end{abstract}

\section{Introduction}
Recent advances in optimization for large language models have explored diverse approaches including adaptive methods \cite{kingma2014adam}, sign-based updates \cite{chen2023symbolic}, and orthogonal weight transformations \cite{li2023muon}. While hybrid optimizers have shown promise by combining spectral processing with sign-based updates \cite{spectrallion}, the interaction between orthogonal constraints and adaptive optimization remains poorly understood. Our work systematically investigates this through OrthoSign, which integrates:

\begin{itemize}
\item Momentum-based orthogonalization via fixed-point iteration
\item Column-wise adaptive scaling
\item Stabilized weight decay
\end{itemize}

\section{Related Work}
Our method builds upon and contrasts with several key developments:

\subsection{Orthogonal Optimization}
Muon \cite{li2023muon} demonstrated the effectiveness of explicit orthogonal constraints.

\subsection{Sign-Based Methods}
Lion \cite{chen2023symbolic} established the viability of sign-based updates.

\subsection{Adaptive Optimization}
AdamW \cite{loshchilov2017decoupled} remains the gold standard, though recent work has shown promise with second-order approaches \cite{sophia}.

\section{Method}
The OrthoSign update rule combines three key components:

\subsection{Orthogonalization}
Given gradient $G_t$ and momentum $M_t$, we compute:

\begin{equation}
X_t = \beta M_{t-1} + (1-\beta)G_t
\end{equation}

\begin{equation}
X_t \leftarrow X_t/\|X_t\|_F
\end{equation}

Then apply $k$ steps of sign-based orthogonalization:

\begin{equation}
X_t \leftarrow \frac{1}{2}(X_t + \text{sgn}(X_t))
\end{equation}

\subsection{Column-wise Scaling}
For each column $j$:

\begin{equation}
s_j = \sqrt{\frac{\|X_{:,j}\|_2}{\frac{1}{d}\sum_{i=1}^d \|X_{:,i}\|_2}}
\end{equation}

\subsection{Update Rule}
The final parameter update becomes:

\begin{equation}
\theta_{t+1} = (1-\lambda)\theta_t - \eta(X_t \odot S)
\end{equation}

where $S$ contains the column scales $s_j$.

\section{Experiments}
We evaluate on FineWeb using a 134M parameter Qwen architecture with:

\begin{itemize}
\item Batch size: 512
\item Max steps: 100,000
\item Learning rates: $10^{-3}$, $10^{-2}$, $10^{-1}$
\item Weight decay: 0.01
\item Orthogonal steps: 1,3,5
\end{itemize}

We track:
\begin{itemize}
\item Training loss curves
\item Gradient norm dynamics
\item Orthogonality error $\|W^TW-I\|_F$
\end{itemize}

\section{Results}
\begin{table}[h]
\centering
\begin{tabular}{lcc}
\toprule
Method & Loss & Orth. Error \\
\midrule
Muon & 3.537 & 0.12 \\
AdamW & 4.927 & - \\
OrthoSign (k=1) & 5.892 & 0.89 \\
OrthoSign (k=3) & 6.584 & 0.45 \\
OrthoSign (k=5) & 7.213 & 0.22 \\
\bottomrule
\end{tabular}
\caption{Performance and orthogonality metrics}
\end{table}

Key findings:
\begin{itemize}
\item More aggressive orthogonalization (higher k) improves orthogonality but harms loss
\item Column scaling appears to interfere with optimization
\item Learning rate sensitivity is 10x higher than baselines
\end{itemize}

\section{Discussion}
The failure modes suggest:

\begin{itemize}
\item Strict orthogonality harms parameter updates
\item Column norms disrupt careful balance
\item Sign updates conflict with momentum
\end{itemize}

\section{Conclusion}
While OrthoSign underperformed, our analysis provides valuable insights for future hybrid optimizers:
\begin{itemize}
\item Gradual orthogonalization may be preferable
\item Separate scaling factors for ortho and adapt components
\item Careful learning rate warmup for constrained optimizers
\end{itemize}

\bibliographystyle{plain}
\bibliography{references}

\end{document}
