Transformer constructor

Transformer({
  1. int vocabSize = 50,
  2. int embedSize = 32,
  3. int blockSize = 8,
  4. int numLayers = 4,
  5. int numHeads = 4,
})

Implementation

Transformer({
  this.vocabSize = 50,
  this.embedSize = 32,
  this.blockSize = 8,
  this.numLayers = 4,
  this.numHeads = 4,
})  :
      // Learnable embedding tables
      tokenEmbeddings = List.generate(
          vocabSize,
          (i) => ValueVector.fromDoubleList(List.generate(
              embedSize, (j) => math.Random().nextDouble() * 0.02 - 0.01))),
      positionEmbeddings = List.generate(
          blockSize,
          (i) => ValueVector.fromDoubleList(List.generate(
              embedSize, (j) => math.Random().nextDouble() * 0.02 - 0.01))),

      // Stack of transformer blocks
      blocks = List.generate(numLayers,
          (i) => TransformerBlock(embedSize, numHeads, masked: true)),

      // Final layers for output
      finalLayerNorm = LayerNorm(embedSize),
      lmHead = Layer.fromNeurons(embedSize, vocabSize);