PythiaBlock constructor

PythiaBlock(
  1. int embedDim,
  2. int numHeads, {
  3. int? ffnDim,
  4. required RopeCache rope,
  5. Device device = Device.CPU,
  6. int seed = 0,
})

Implementation

PythiaBlock(
  int embedDim,
  int numHeads, {
  int? ffnDim,
  required RopeCache rope,
  Device device = Device.CPU,
  int seed = 0,
}) : inputLn = LayerNorm(embedDim, device: device),
     postAttnLn = LayerNorm(embedDim, device: device),
     attn = MultiHeadAttention(
       embedDim,
       numHeads,
       bias: true,
       device: device,
       seed: seed,
     ),
     ffn1 = Linear(
       embedDim,
       ffnDim ?? 4 * embedDim,
       bias: true,
       device: device,
       seed: seed + 10000,
     ),
     ffn2 = Linear(
       ffnDim ?? 4 * embedDim,
       embedDim,
       bias: true,
       device: device,
       seed: seed + 20000,
     ) {
  attn.rope = rope;
}