embedderConfigFor function
Where config runs, as the embedder's own config: the bundle's NPU
stamp, read from its bytes, decides; nothing moves to another backend on
its own.
Implementation
Future<LiteRtLmEmbeddingConfig> embedderConfigFor(
EmbeddingEngineConfig config,
) async {
if (config.backend case CpuBackend(
:final threadCount?,
) when threadCount < 1) {
throw ArgumentError.value(threadCount, 'threadCount', 'must be 1 or more');
}
final path = config.modelPath;
final file = path.split(RegExp(r'[/\\]')).last;
Never refuse(String message) => throw EmbedderLoadException(message);
// A file that cannot be read is a load failure, as every refusal here.
Future<T> read<T>(Future<T> Function() body) async {
try {
return await body();
} on EmbedderLoadException {
rethrow;
} on Object catch (e) {
if (e is! FileSystemException &&
e is! FormatException &&
e is! RangeError &&
e is! LiteRtLmBundleException) {
rethrow;
}
throw EmbedderLoadException(
'$file is not a readable .litertlm bundle: $e',
cause: e,
);
}
}
final target = await read(() => readLiteRtLmNpuTarget(path));
final backend = config.backend;
final abi = currentEngineHost.abi;
var gpu = false;
String? dispatchDir;
NpuStamp? npuTarget;
switch (backend) {
case CpuBackend():
if (target != null) {
refuse('$file is compiled for the $target NPU and runs only there.');
}
case GpuBackend():
if (target != null) {
refuse('$file is compiled for the $target NPU and runs only there.');
}
if (liteRtLmEmbeddingGpuRefusedAbis.contains(abi)) {
refuse(
"EmbeddingGemma 2 on the GPU ends the process on $abi with this "
"release's native libraries; load $file on the CPU.",
);
}
// Never moved to the CPU: the GPU is honoured where it was measured,
// for the bundles that ran there, and refused elsewhere.
if (!liteRtLmEmbeddingGpuAbis.contains(abi)) {
refuse(
'EmbeddingGemma 2 runs on the GPU on '
'${liteRtLmEmbeddingGpuAbis.join(', ')} only, and this is $abi; '
'load $file on the CPU.',
);
}
final towers = await read(() async => readLiteRtLmMediaTowers(path));
final bundle = LiteRtLmEg2Bundle.of(
vision: towers.vision,
audio: towers.audio,
);
final measured = liteRtLmEmbeddingGpuMeasured[abi] ?? const {};
if (!measured.contains(bundle)) {
refuse(
'EmbeddingGemma 2 on the GPU on $abi has run with '
'${measured.map((b) => b.label).join(' and ')} only; $file is '
'${bundle.label}, not run on this GPU yet. Load it on the CPU.',
);
}
if (config.visionBackend != null && !towers.vision) {
refuse('$file has no vision tower: it takes no images.');
}
gpu = true;
case final NpuBackend npu:
final dir = npu.dispatchDir;
if (target == null) {
refuse(
'$file is the generic EmbeddingGemma 2 bundle: it has no NPU graph '
"and runs on the CPU only. Load the bundle compiled for this "
"device's SoC to embed on the NPU.",
);
}
final vendor = npuVendorOf(target);
if (vendor == null || !NpuStack.forAbi(abi).contains(vendor)) {
refuse(
'$file is compiled for the $target NPU, which no dispatch on $abi '
'runs.',
);
}
npuTarget = target;
// The stack and the device are checked as for Engine.create, a
// prepared directory or not; the directory only replaces the one the
// host would prepare.
final host = npuHostWithDispatchDir(
currentEngineHost.npuHost(npuPlatformOf(npu)),
dir,
);
if (await chooseNpuVendor(target: target, host: host) case NpuRefused(
:final reason,
)) {
refuse('$file needs the $target NPU, and $reason.');
}
dispatchDir = dir ?? await host.prepareDispatchDir(vendor);
}
// The towers run where the bundle puts them: vision on the NPU beside an
// NPU text tower and on the CPU otherwise, audio on the CPU everywhere
// (LiteRT-LM's EG2 audio encoder is compiled for the CPU and the GPU, and
// its GPU is not measured). Another backend is refused, never moved.
final textKind = backendKindOf(backend);
final visionKind = textKind == BackendKind.npu
? BackendKind.npu
: BackendKind.cpu;
if (config.visionBackend case final vision?
when backendKindOf(vision) != visionKind) {
refuse(
"$file's vision tower runs on the ${visionKind.name.toUpperCase()} "
'beside a ${textKind.name.toUpperCase()} text tower, not on the '
'${backendKindOf(vision).name.toUpperCase()}: pass visionBackend: '
'Backend.${visionKind.name}().',
);
}
if (config.audioBackend case final audio? when audio is! CpuBackend) {
refuse(
"$file's audio tower runs on the CPU only, not on the "
'${backendKindOf(audio).name.toUpperCase()}: pass audioBackend: '
'Backend.cpu().',
);
}
final declared = {
if (config.visionBackend != null) EmbedderModality.image,
if (config.audioBackend != null) EmbedderModality.audio,
};
// The towers a bundle has load with the text tower: vision on the CPU beside
// a GPU text tower, audio on the CPU everywhere (LiteRT-LM's EG2 audio
// encoder is compiled for cpu/gpu only).
final onAccelerator = gpu || npuTarget != null;
final towers = onAccelerator
? await read(() async => readLiteRtLmMediaTowers(path))
: (vision: false, audio: false);
return LiteRtLmEmbeddingConfig(
modelPath: path,
cacheBase: config.cacheDir,
declaredMedia: declared,
numThreads: switch (backend) {
CpuBackend(:final threadCount) => threadCount,
_ => null,
},
visionTokensPerImage:
config.visionTokensPerImage ?? liteRtLmDefaultVisionTokensPerImage,
requireWeightCache: config.requireWeightCache,
npuTarget: npuTarget,
npuDispatchDir: dispatchDir,
gpu: gpu,
visionTowerOnCpu: gpu && towers.vision,
audioTowerOnCpu: onAccelerator && towers.audio,
);
}