decode method

Future<List<KeywordDetection>> decode(KwsEncoderOutput encoder)

Implementation

Future<List<KeywordDetection>> decode(KwsEncoderOutput encoder) async {
  if (encoder.values.length != encoder.frameCount * joinerDimension) {
    throw StateError('Invalid encoder output shape');
  }
  final detections = <KeywordDetection>[];

  for (var frame = 0; frame < encoder.frameCount; frame++) {
    final previous = _hypotheses.values.toList(growable: false);
    final contexts = Int32List(previous.length * contextSize);
    for (var i = 0; i < previous.length; i++) {
      final tokens = previous[i].tokens;
      contexts[i * 2] = tokens[tokens.length - 2];
      contexts[i * 2 + 1] = tokens.last;
    }
    final decoderVectors = await _backend.runDecoder(contexts);
    final encoderVectors = Float32List(previous.length * joinerDimension);
    final frameStart = frame * joinerDimension;
    final frameVector = encoder.values.sublist(
      frameStart,
      frameStart + joinerDimension,
    );
    for (var i = 0; i < previous.length; i++) {
      encoderVectors.setRange(
        i * joinerDimension,
        (i + 1) * joinerDimension,
        frameVector,
      );
    }
    final logits = await _backend.runJoiner(encoderVectors, decoderVectors);
    if (logits.length != previous.length * vocabSize) {
      throw StateError('Invalid joiner output shape');
    }
    _logSoftmaxRows(logits, vocabSize);

    final candidates = <_Candidate>[];
    for (
      var hypothesisIndex = 0;
      hypothesisIndex < previous.length;
      hypothesisIndex++
    ) {
      final base = hypothesisIndex * vocabSize;
      for (var token = 0; token < vocabSize; token++) {
        candidates.add(
          _Candidate(
            hypothesisIndex,
            token,
            logits[base + token] + previous[hypothesisIndex].logProbability,
            logits[base + token],
          ),
        );
      }
    }
    candidates.sort((left, right) => right.combined.compareTo(left.combined));

    final next = <String, KwsHypothesis>{};
    final count = math.min(maxActivePaths, candidates.length);
    for (var i = 0; i < count; i++) {
      final candidate = candidates[i];
      final hypothesis = previous[candidate.hypothesisIndex].copy();
      var contextScore = 0.0;
      if (candidate.token != blankId && candidate.token != unknownId) {
        hypothesis.tokens.add(candidate.token);
        hypothesis.timestamps.add(frame + _frameOffset);
        hypothesis.tokenProbabilities.add(math.exp(candidate.acoustic));
        hypothesis.trailingBlanks = 0;
        final step = _graph.forward(hypothesis.contextState, candidate.token);
        contextScore = step.score;
        hypothesis.contextState = step.state;
        if (hypothesis.contextState.token == -1) {
          hypothesis.tokens
            ..clear()
            ..addAll(const [-1, blankId]);
          hypothesis.timestamps.clear();
          hypothesis.tokenProbabilities.clear();
        }
      } else {
        hypothesis.trailingBlanks++;
      }
      hypothesis.logProbability = candidate.combined + contextScore;
      _addOrMerge(next, hypothesis);
    }

    final best = _best(next.values);
    final matched = _graph.matched(best.contextState);
    if (matched != null &&
        best.trailingBlanks > trailingBlankFrames &&
        best.tokenProbabilities.length >= matched.level) {
      final probabilities = best.tokenProbabilities.sublist(
        best.tokenProbabilities.length - matched.level,
      );
      final mean = probabilities.reduce((a, b) => a + b) / matched.level;
      if (mean >= matched.acousticThreshold) {
        final timestamps = best.timestamps.sublist(
          best.timestamps.length - matched.level,
        );
        detections.add(
          KeywordDetection(
            phrase: matched.phrase,
            detectedAt: Duration(milliseconds: (frame + _frameOffset) * 40),
            tokenTimestamps: timestamps
                .map((frame) => Duration(milliseconds: frame * 40))
                .toList(growable: false),
            meanTokenProbability: mean,
          ),
        );
        final empty = KwsHypothesis(
          tokens: <int>[-1, blankId],
          logProbability: 0,
          contextState: _graph.root,
        );
        next
          ..clear()
          ..[empty.key] = empty;
      }
    }
    _hypotheses = next;
  }
  _frameOffset += encoder.frameCount;
  return detections;
}