embedderConfigFor function

Where config runs, as the embedder's own config: the bundle's NPU stamp, read from its bytes, decides; nothing moves to another backend on its own.

Implementation

Future<LiteRtLmEmbeddingConfig> embedderConfigFor(
  EmbeddingEngineConfig config,
) async {
  if (config.backend case CpuBackend(
    :final threadCount?,
  ) when threadCount < 1) {
    throw ArgumentError.value(threadCount, 'threadCount', 'must be 1 or more');
  }
  final path = config.modelPath;
  final file = path.split(RegExp(r'[/\\]')).last;
  Never refuse(String message) => throw EmbedderLoadException(message);
  // A file that cannot be read is a load failure, as every refusal here.
  Future<T> read<T>(Future<T> Function() body) async {
    try {
      return await body();
    } on EmbedderLoadException {
      rethrow;
    } on Object catch (e) {
      if (e is! FileSystemException &&
          e is! FormatException &&
          e is! RangeError &&
          e is! LiteRtLmBundleException) {
        rethrow;
      }
      throw EmbedderLoadException(
        '$file is not a readable .litertlm bundle: $e',
        cause: e,
      );
    }
  }

  final target = await read(() => readLiteRtLmNpuTarget(path));
  final backend = config.backend;
  final abi = currentEngineHost.abi;
  var gpu = false;
  String? dispatchDir;
  NpuStamp? npuTarget;
  switch (backend) {
    case CpuBackend():
      if (target != null) {
        refuse('$file is compiled for the $target NPU and runs only there.');
      }
    case GpuBackend():
      if (target != null) {
        refuse('$file is compiled for the $target NPU and runs only there.');
      }
      if (liteRtLmEmbeddingGpuRefusedAbis.contains(abi)) {
        refuse(
          "EmbeddingGemma 2 on the GPU ends the process on $abi with this "
          "release's native libraries; load $file on the CPU.",
        );
      }
      // Never moved to the CPU: the GPU is honoured where it was measured,
      // for the bundles that ran there, and refused elsewhere.
      if (!liteRtLmEmbeddingGpuAbis.contains(abi)) {
        refuse(
          'EmbeddingGemma 2 runs on the GPU on '
          '${liteRtLmEmbeddingGpuAbis.join(', ')} only, and this is $abi; '
          'load $file on the CPU.',
        );
      }
      final towers = await read(() async => readLiteRtLmMediaTowers(path));
      final bundle = LiteRtLmEg2Bundle.of(
        vision: towers.vision,
        audio: towers.audio,
      );
      final measured = liteRtLmEmbeddingGpuMeasured[abi] ?? const {};
      if (!measured.contains(bundle)) {
        refuse(
          'EmbeddingGemma 2 on the GPU on $abi has run with '
          '${measured.map((b) => b.label).join(' and ')} only; $file is '
          '${bundle.label}, not run on this GPU yet. Load it on the CPU.',
        );
      }
      if (config.visionBackend != null && !towers.vision) {
        refuse('$file has no vision tower: it takes no images.');
      }
      gpu = true;
    case final NpuBackend npu:
      final dir = npu.dispatchDir;
      if (target == null) {
        refuse(
          '$file is the generic EmbeddingGemma 2 bundle: it has no NPU graph '
          "and runs on the CPU only. Load the bundle compiled for this "
          "device's SoC to embed on the NPU.",
        );
      }
      final vendor = npuVendorOf(target);
      if (vendor == null || !NpuStack.forAbi(abi).contains(vendor)) {
        refuse(
          '$file is compiled for the $target NPU, which no dispatch on $abi '
          'runs.',
        );
      }
      npuTarget = target;
      // The stack and the device are checked as for Engine.create, a
      // prepared directory or not; the directory only replaces the one the
      // host would prepare.
      final host = npuHostWithDispatchDir(
        currentEngineHost.npuHost(npuPlatformOf(npu)),
        dir,
      );
      if (await chooseNpuVendor(target: target, host: host) case NpuRefused(
        :final reason,
      )) {
        refuse('$file needs the $target NPU, and $reason.');
      }
      dispatchDir = dir ?? await host.prepareDispatchDir(vendor);
  }
  // The towers run where the bundle puts them: vision on the NPU beside an
  // NPU text tower and on the CPU otherwise, audio on the CPU everywhere
  // (LiteRT-LM's EG2 audio encoder is compiled for the CPU and the GPU, and
  // its GPU is not measured). Another backend is refused, never moved.
  final textKind = backendKindOf(backend);
  final visionKind = textKind == BackendKind.npu
      ? BackendKind.npu
      : BackendKind.cpu;
  if (config.visionBackend case final vision?
      when backendKindOf(vision) != visionKind) {
    refuse(
      "$file's vision tower runs on the ${visionKind.name.toUpperCase()} "
      'beside a ${textKind.name.toUpperCase()} text tower, not on the '
      '${backendKindOf(vision).name.toUpperCase()}: pass visionBackend: '
      'Backend.${visionKind.name}().',
    );
  }
  if (config.audioBackend case final audio? when audio is! CpuBackend) {
    refuse(
      "$file's audio tower runs on the CPU only, not on the "
      '${backendKindOf(audio).name.toUpperCase()}: pass audioBackend: '
      'Backend.cpu().',
    );
  }
  final declared = {
    if (config.visionBackend != null) EmbedderModality.image,
    if (config.audioBackend != null) EmbedderModality.audio,
  };
  // The towers a bundle has load with the text tower: vision on the CPU beside
  // a GPU text tower, audio on the CPU everywhere (LiteRT-LM's EG2 audio
  // encoder is compiled for cpu/gpu only).
  final onAccelerator = gpu || npuTarget != null;
  final towers = onAccelerator
      ? await read(() async => readLiteRtLmMediaTowers(path))
      : (vision: false, audio: false);
  return LiteRtLmEmbeddingConfig(
    modelPath: path,
    cacheBase: config.cacheDir,
    declaredMedia: declared,
    numThreads: switch (backend) {
      CpuBackend(:final threadCount) => threadCount,
      _ => null,
    },
    visionTokensPerImage:
        config.visionTokensPerImage ?? liteRtLmDefaultVisionTokensPerImage,
    requireWeightCache: config.requireWeightCache,
    npuTarget: npuTarget,
    npuDispatchDir: dispatchDir,
    gpu: gpu,
    visionTowerOnCpu: gpu && towers.vision,
    audioTowerOnCpu: onAccelerator && towers.audio,
  );
}