TransformerEncoder constructor

TransformerEncoder({ required int vocabSize, required int embedSize, required int blockSize, int numLayers = 6, int numHeads = 8, })

Implementation

TransformerEncoder({
  required this.vocabSize,
  required this.embedSize,
  required this.blockSize,
  this.numLayers = 6,
  this.numHeads = 8,
}) : assert(
       embedSize % numHeads == 0,
       "embedSize must be divisible by numHeads",
     ),
     // Initialize directly on GPU with scaling
     wte = Tensor.random([vocabSize, embedSize], scale: 0.02),
     wpe = Tensor.random([blockSize, embedSize], scale: 0.02),
     blocks = List.generate(
       numLayers,
       (i) => TransformerEncoderBlock(embedSize, numHeads, blockSize),
     ),
     finalLayerNorm = LayerNorm(embedSize);