GPTModel constructor

GPTModel(
  1. GPTConfig cfg
)

Implementation

GPTModel(this.cfg)
    : tokEmb = Embedding(cfg.vocabSize, cfg.dModel),
      posEmb = Embedding(cfg.maxSeqLen, cfg.dModel),
      blocks = List.generate(
          cfg.nLayers,
          (_) => TransformerBlock(cfg.dModel, cfg.nHeads,
              ffnMult: cfg.ffnMult, rope: cfg.rope)),
      lnF = LayerNorm(cfg.dModel),
      head = LinearLMHead(cfg.dModel, cfg.vocabSize),
      _kv = List.generate(cfg.nLayers, (_) => KVCache(cfg.nHeads, cfg.dModel ~/ cfg.nHeads)) {
  if (cfg.tieWeights) {
    head.tieTo(tokEmb);
  }
}