TransformerEncoderBlock constructor

TransformerEncoderBlock(
  1. int embedSize,
  2. int numHeads,
  3. int maxSeqLen
)

Implementation

TransformerEncoderBlock(this.embedSize, int numHeads, int maxSeqLen)
  : attention = MultiHeadAFT(
      numHeads,
      embedSize,
      maxSeqLen,
      masked: false, // Encoder is bidirectional, not causal
    ),
    // FFN typically expands and then contracts;
    // here using your Layer class for computation.
    ffn = Layer(embedSize, embedSize, useGelu: true),
    ln1 = LayerNorm(embedSize),
    ln2 = LayerNorm(embedSize);