call method

Tensor call(
  1. Tensor x
)

Implementation

Tensor call(Tensor x) {
  if (variant == ExpertVariant.swiGlu) {
    // SwiGLU: w2(silu(w1(x)) * w3(x)). Always uses silu — the gate
    // itself is the nonlinearity, per DeepSeek-V3 / Mixtral.
    final gate = w1(x);
    final up = w3!(x);
    final silu = gate * gate.sigmoid();
    return w2(silu * up);
  }
  final h = w1(x);
  final a = switch (activation) {
    ExpertActivation.relu => h.relu(),
    ExpertActivation.silu => h * h.sigmoid(),
  };
  return w2(a);
}