gelu method

Value gelu()

Implementation

Value gelu() {
  final cdf = 0.5 * (1 + tanH(data / math.sqrt(2)));
  final out = Value(data * cdf, {this}, 'GELU');
  out._backward = () {
    final derivative = 0.5 * (1 + tanH(data / math.sqrt(2))) +
        (data * (1 - tanH(data / math.sqrt(2)) * tanH(data / math.sqrt(2)))) /
            (2 * math.sqrt(2 * math.pi));
    grad += derivative * out.grad;
  };
  return out;
}