Expert constructor

Expert(
  1. int dim,
  2. int hiddenDim, {
  3. Device device = Device.CPU,
  4. int seed = 0,
  5. ExpertActivation activation = ExpertActivation.relu,
  6. ExpertVariant variant = ExpertVariant.mlp,
})

Implementation

Expert(
  this.dim,
  this.hiddenDim, {
  Device device = Device.CPU,
  int seed = 0,
  this.activation = ExpertActivation.relu,
  this.variant = ExpertVariant.mlp,
}) : w1 = Linear(dim, hiddenDim, device: device, seed: seed),
     w2 = Linear(hiddenDim, dim, device: device, seed: seed + 1),
     w3 = variant == ExpertVariant.swiGlu
         ? Linear(dim, hiddenDim, device: device, seed: seed + 2)
         : null;