TransformerDecoder constructor

TransformerDecoder({
  1. int vocabSize = 50,
  2. int embedSize = 32,
  3. int blockSize = 8,
  4. int numLayers = 4,
  5. int numHeads = 4,
  6. int encoderEmbedSize = 64,
})

Implementation

TransformerDecoder({
  this.vocabSize = 50,
  this.embedSize = 32,
  this.blockSize = 8,
  this.numLayers = 4,
  this.numHeads = 4,
  this.encoderEmbedSize = 64, // NEW: Must match the encoder's embedSize
})  : assert(embedSize % numHeads == 0),
      tokenEmbeddings = List.generate(
          vocabSize,
          (i) => ValueVector.fromDoubleList(List.generate(
              embedSize, (j) => math.Random().nextDouble() * 0.02 - 0.01))),
      positionEmbeddings = List.generate(
          blockSize,
          (i) => ValueVector.fromDoubleList(List.generate(
              embedSize, (j) => math.Random().nextDouble() * 0.02 - 0.01))),
      // Each decoder block needs the encoder's embed size for cross-attention
      blocks = List.generate(
          numLayers,
          (i) =>
              TransformerDecoderBlock(embedSize, numHeads, encoderEmbedSize)),
      finalLayerNorm = LayerNorm(embedSize),
      lmHead = Layer.fromNeurons(embedSize, vocabSize);