ParcaeAdapter: rho(A)<1 by construction (ZOH negative-diag), rho logging in rec arms, pre-registration item 12
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -127,6 +127,58 @@ class RecurrentAdapter(nn.Module):
|
||||
+ self.mlp(torch.cat([e32, s_hat], dim=-1)))
|
||||
return out.to(dt)
|
||||
|
||||
def rho(self):
|
||||
"""Spectral radius of A (power iteration) — the stability dial."""
|
||||
with torch.no_grad():
|
||||
v = torch.randn(self.A.weight.shape[0],
|
||||
device=self.A.weight.device)
|
||||
for _ in range(30):
|
||||
v = self.A.weight.T @ (self.A.weight @ v)
|
||||
v = v / (v.norm() + 1e-12)
|
||||
return (self.A.weight @ v).norm().item()
|
||||
|
||||
|
||||
class ParcaeAdapter(RecurrentAdapter):
|
||||
"""Recurrent adapter with ρ(A) < 1 GUARANTEED by construction
|
||||
(Parcae parameterization, Prairie et al. 2026): A is the ZOH
|
||||
discretization of a continuous negative-diagonal system,
|
||||
A = exp(−Δt·exp(a)) with learned per-dim a and scalar log-Δt, so every
|
||||
entry lies in (0,1) regardless of optimizer noise. Init matches the
|
||||
fixed merge exactly: a = log(−log α), Δt = 1 → A = α·I."""
|
||||
|
||||
def __init__(self, d=1536, hidden=512, alpha=ALPHA, sigma=1.0):
|
||||
nn.Module.__init__(self)
|
||||
import math as _m
|
||||
self.log_a = nn.Parameter(
|
||||
torch.full((d,), _m.log(-_m.log(alpha))))
|
||||
self.log_dt = nn.Parameter(torch.zeros(()))
|
||||
self.B = nn.Linear(d, d, bias=False)
|
||||
with torch.no_grad():
|
||||
self.B.weight.copy_((1 - alpha) * torch.eye(d))
|
||||
self.mlp = nn.Sequential(
|
||||
nn.Linear(2 * d, hidden), nn.GELU(), nn.Linear(hidden, d)
|
||||
)
|
||||
nn.init.zeros_(self.mlp[2].weight)
|
||||
nn.init.zeros_(self.mlp[2].bias)
|
||||
self.sigma = sigma
|
||||
|
||||
def A_diag(self):
|
||||
return torch.exp(-torch.exp(self.log_dt) * torch.exp(self.log_a))
|
||||
|
||||
def forward(self, e, s):
|
||||
dt = e.dtype
|
||||
e32, s32 = e.float(), s.float()
|
||||
s_hat = s32 * (
|
||||
e32.norm(dim=-1, keepdim=True) / (s32.norm(dim=-1, keepdim=True) + 1e-6)
|
||||
)
|
||||
out = (self.A_diag().to(e32.dtype) * s_hat + self.B(e32)
|
||||
+ self.mlp(torch.cat([e32, s_hat], dim=-1)))
|
||||
return out.to(dt)
|
||||
|
||||
def rho(self):
|
||||
with torch.no_grad():
|
||||
return self.A_diag().max().item()
|
||||
|
||||
|
||||
class BandLooper:
|
||||
"""Capture layer-call kwargs once per forward, then re-run L14-30 manually."""
|
||||
|
||||
Reference in New Issue
Block a user