openvino_genai 0.2.0

On-device LLM and vision-language (VLM) inference for Flutter via Intel OpenVINO GenAI: streaming chat and image description on CPU/GPU, hardened load/fallback. Windows x64; unofficial.

example/lib/main.dart

import 'dart:async';
import 'dart:io';
import 'dart:typed_data';

import 'package:flutter/material.dart';
import 'package:image/image.dart' as img;
import 'package:openvino_genai/openvino_genai.dart';

void main() {
  // Optional device-ladder override for the LLM lane, e.g. "NPU,GPU,CPU"
  // (default GPU,CPU). The VLM lane defaults to CPU; override with
  // OPENVINO_GENAI_EXAMPLE_VLM_DEVICES.
  final devices = Platform.environment['OPENVINO_GENAI_EXAMPLE_DEVICES'];
  if (devices != null && devices.trim().isNotEmpty) {
    OpenVinoGenAiService.devicePreference = _splitDevices(devices);
  }
  final vlmDevices = Platform.environment['OPENVINO_GENAI_EXAMPLE_VLM_DEVICES'];
  if (vlmDevices != null && vlmDevices.trim().isNotEmpty) {
    OpenVinoVlmService.devicePreference = _splitDevices(vlmDevices);
  }
  // Doctor mode: bind BOTH bridges, print diagnostics, exit 0/1. Needs no
  // model and no GPU — CI uses it to prove the bundled DLL set loads and
  // all 26 exports across both bridge DLLs bind on a clean machine.
  if (Platform.environment['OPENVINO_GENAI_EXAMPLE_DOCTOR'] == '1' &&
      !Platform.environment.containsKey('FLUTTER_TEST')) {
    unawaited(_runDoctorMode());
    return;
  }
  runApp(const ExampleApp());
}

List<String> _splitDevices(String csv) => [
      for (final d in csv.split(','))
        if (d.trim().isNotEmpty) d.trim(),
    ];

Future<void> _runDoctorMode() async {
  final llm = await OpenVinoGenAiService().doctor();
  final vlm = await OpenVinoVlmService().doctor();
  stdout.writeln('[doctor] LLM $llm');
  stdout.writeln('[doctor] VLM $vlm');
  // CPU must appear in the enumeration: that proves openvino.dll and
  // openvino_c.dll actually loaded and answered, not just that the bridge
  // DLLs bound. Both bridges must report the same version and runtime pin —
  // they are compiled from one CMake configure against one archive.
  bool healthy(OpenVinoGenAiDiagnostics d) =>
      d.dllFound &&
      d.bridgeBound &&
      (d.availableDevices?.contains('CPU') ?? false);
  final ok = healthy(llm) &&
      healthy(vlm) &&
      llm.bridgeVersion == vlm.bridgeVersion &&
      llm.runtimePin == vlm.runtimePin;
  stdout.writeln(ok ? '[doctor] PASS' : '[doctor] FAIL');
  exit(ok ? 0 : 1);
}

/// Self-test lanes for CI / smoke testing on a build machine. Whichever of
/// these are set run in order, and the process exits 0 only if all pass:
///  * `OPENVINO_GENAI_EXAMPLE_MODEL`     — LLM: load, one greedy reply.
///  * `OPENVINO_GENAI_EXAMPLE_VLM_MODEL` — VLM: load, synthetic-image
///    descriptions (colour probe + 640x480 frame), warm repeat, unload/reload.
///  * `OPENVINO_GENAI_EXAMPLE_VLM_IMAGE` — optional real photo for the VLM
///    lane (decoded + resized with package:image).
Future<void> runAutoTests({
  String? llmModel,
  String? vlmModel,
  String? vlmImage,
}) async {
  var ok = true;
  if (llmModel != null && llmModel.isNotEmpty) {
    ok = await runLlmAutoTest(llmModel) && ok;
  }
  if (vlmModel != null && vlmModel.isNotEmpty) {
    ok = await runVlmAutoTest(vlmModel, imagePath: vlmImage) && ok;
  }
  stdout.writeln(ok ? '[autotest] PASS' : '[autotest] FAIL');
  exit(ok ? 0 : 1);
}

/// LLM lane: load [modelDir], one short greedy generation.
Future<bool> runLlmAutoTest(String modelDir) async {
  final llm = OpenVinoGenAiService();
  try {
    stdout.writeln('[autotest] doctor:\n${await llm.doctor()}');
    stdout.writeln('[autotest] loading $modelDir');
    final watch = Stopwatch()..start();
    await llm.loadModel(modelDir);
    stdout.writeln(
        '[autotest] loaded on ${llm.activeDevice} in ${watch.elapsed}');
    final reply = await llm.generateText(
      [
        const LlmMessage.system('You are a terse assistant.'),
        const LlmMessage.user('Reply with the single word: OK'),
      ],
      params: LlmGenerationParams.greedy.copyWith(maxTokens: 16),
    );
    stdout.writeln('[autotest] reply: $reply');
    await llm.unloadModel();
    stdout.writeln('[autotest] LLM lane PASS');
    return true;
  } on Object catch (e) {
    stdout.writeln('[autotest] LLM lane FAIL: $e');
    return false;
  }
}

/// True when [text] names both colours as whole words (a substring check
/// would accept "colored" / "centered" / "covered").
bool namesRedAndBlue(String text) {
  final lower = text.toLowerCase();
  return RegExp(r'\bred\b').hasMatch(lower) &&
      RegExp(r'\bblue\b').hasMatch(lower);
}

/// A [width]x[height] RGB8 test card: left half pure red, right half pure
/// blue — the probe the original C++ bridge was validated with.
VlmImage colourProbe({int width = 224, int height = 224}) {
  final bytes = Uint8List(width * height * 3);
  for (var y = 0; y < height; y++) {
    for (var x = 0; x < width; x++) {
      final i = (y * width + x) * 3;
      if (x < width ~/ 2) {
        bytes[i] = 255;
      } else {
        bytes[i + 2] = 255;
      }
    }
  }
  return VlmImage.rgb8(bytes: bytes, width: width, height: height);
}

/// A 640x480 synthetic "camera frame": a horizontal gradient with a black
/// rectangle — realistic encode cost without a real camera.
VlmImage syntheticFrame() {
  const width = 640, height = 480;
  final bytes = Uint8List(width * height * 3);
  for (var y = 0; y < height; y++) {
    for (var x = 0; x < width; x++) {
      final i = (y * width + x) * 3;
      final inBox = x > 200 && x < 440 && y > 150 && y < 330;
      if (!inBox) {
        bytes[i] = (x * 255 ~/ width);
        bytes[i + 1] = 128;
        bytes[i + 2] = (y * 255 ~/ height);
      }
    }
  }
  return VlmImage.rgb8(bytes: bytes, width: width, height: height);
}

/// VLM lane: load [modelDir] (CPU by default), describe synthetic images,
/// optionally a real photo at [imagePath], then unload and reload.
Future<bool> runVlmAutoTest(String modelDir, {String? imagePath}) async {
  final vlm = OpenVinoVlmService();
  const probeParams =
      LlmGenerationParams(maxTokens: 128, temperature: 0.0, topP: 1.0);
  try {
    stdout.writeln('[vlm] doctor:\n${await vlm.doctor()}');
    stdout.writeln('[vlm] loading $modelDir');
    final watch = Stopwatch()..start();
    await vlm.loadModel(modelDir);
    stdout.writeln('[vlm] loaded on ${vlm.activeDevice} in ${watch.elapsed}');

    // 1. Colour probe: the answer must name both colours.
    final probe = colourProbe();
    final colours = await vlm.describeImage(
      probe,
      prompt: 'What colours are in this picture and where are they?',
      params: probeParams,
    );
    stdout.writeln('[vlm] colours: ${colours.trim()}');
    stdout.writeln('[vlm] first describe: ttft=${vlm.lastTimeToFirstToken} '
        'total=${vlm.lastGenerateDuration}');
    if (!namesRedAndBlue(colours)) {
      stdout.writeln('[vlm] FAIL: colour probe did not name red and blue');
      return false;
    }

    // 2. Warm repeat of the same image — pipeline reusable, no state leak:
    //    the answer must still be right and the run must not be slower.
    final firstDuration = vlm.lastGenerateDuration!;
    final again = await vlm.describeImage(probe,
        prompt: 'What colours are in this picture and where are they?',
        params: probeParams);
    final second = vlm.lastGenerateDuration!;
    stdout.writeln('[vlm] second describe: total=$second');
    if (!namesRedAndBlue(again)) {
      stdout.writeln('[vlm] FAIL: warm repeat answered "$again"');
      return false;
    }
    if (second > firstDuration * 1.5 + const Duration(seconds: 2)) {
      stdout.writeln('[vlm] FAIL: warm describe slower than 1.5x the first');
      return false;
    }

    // 3. Camera-sized frame.
    final frameText = await vlm.describeImage(
      syntheticFrame(),
      prompt: 'Describe this image in one sentence.',
      params: probeParams,
    );
    stdout.writeln('[vlm] 640x480 frame: ${frameText.trim()} '
        '(ttft=${vlm.lastTimeToFirstToken} total=${vlm.lastGenerateDuration})');
    if (frameText.trim().isEmpty) {
      stdout.writeln('[vlm] FAIL: empty description for the 640x480 frame');
      return false;
    }

    // 4. Optional real photo.
    if (imagePath != null && imagePath.isNotEmpty) {
      final decoded = img.decodeImage(await File(imagePath).readAsBytes());
      if (decoded == null) {
        stdout.writeln('[vlm] FAIL: could not decode $imagePath');
        return false;
      }
      final longest = decoded.width > decoded.height
          ? decoded.width
          : decoded.height;
      final resized = longest > 1024
          ? img.copyResize(decoded,
              width: decoded.width > decoded.height ? 1024 : null,
              height: decoded.height >= decoded.width ? 1024 : null)
          : decoded;
      // Normalise 16-bit / paletted / RGBA sources to packed 8-bit RGB —
      // getBytes only converts channel order, not the sample format.
      final rgb8 = resized.convert(format: img.Format.uint8, numChannels: 3);
      final rgb = rgb8.getBytes(order: img.ChannelOrder.rgb);
      final photo =
          VlmImage.rgb8(bytes: rgb, width: rgb8.width, height: rgb8.height);
      final text = await vlm.describeImage(
        photo,
        prompt: 'Describe this picture in detail. If it contains text, '
            'transcribe it.',
        params: probeParams,
      );
      stdout.writeln('[vlm] photo ${resized.width}x${resized.height}: '
          '${text.trim()} (total=${vlm.lastGenerateDuration})');
      if (text.trim().isEmpty) {
        stdout.writeln('[vlm] FAIL: empty description for the photo');
        return false;
      }
    }

    // 5. Unload / reload.
    await vlm.unloadModel();
    if (vlm.isRunning || vlm.isEngineWedged) {
      stdout.writeln('[vlm] FAIL: unload left the lane running or wedged');
      return false;
    }
    await vlm.loadModel(modelDir);
    await vlm.unloadModel();
    stdout.writeln('[vlm] VLM lane PASS');
    return true;
  } on Object catch (e) {
    stdout.writeln('[vlm] VLM lane FAIL: $e');
    return false;
  }
}

/// Minimal demo: pick an OpenVINO IR model directory, load it, chat.
class ExampleApp extends StatelessWidget {
  const ExampleApp({super.key});

  @override
  Widget build(BuildContext context) => MaterialApp(
        title: 'openvino_genai example',
        theme: ThemeData(colorSchemeSeed: Colors.indigo, useMaterial3: true),
        home: const DemoPage(),
      );
}

/// One page: model controls on top, streamed chat output below.
class DemoPage extends StatefulWidget {
  const DemoPage({super.key});

  @override
  State<DemoPage> createState() => _DemoPageState();
}

class _DemoPageState extends State<DemoPage> {
  final _llm = OpenVinoGenAiService();
  final _modelDir = TextEditingController();
  final _prompt = TextEditingController(text: 'Why is the sky blue?');
  final _output = StringBuffer();

  String _status = 'No model loaded.';
  bool _busy = false;
  StreamSubscription<String>? _generation;

  @override
  void initState() {
    super.initState();
    final env = Platform.environment;
    final llmModel = env['OPENVINO_GENAI_EXAMPLE_MODEL'];
    final vlmModel = env['OPENVINO_GENAI_EXAMPLE_VLM_MODEL'];
    final hasLane = (llmModel != null && llmModel.isNotEmpty) ||
        (vlmModel != null && vlmModel.isNotEmpty);
    // FLUTTER_TEST guard: with the env vars exported in the same shell,
    // `flutter test` would otherwise trigger real FFI loads and exit() —
    // hard-killing the test harness.
    if (hasLane && !env.containsKey('FLUTTER_TEST')) {
      if (llmModel != null) _modelDir.text = llmModel;
      unawaited(runAutoTests(
        llmModel: llmModel,
        vlmModel: vlmModel,
        vlmImage: env['OPENVINO_GENAI_EXAMPLE_VLM_IMAGE'],
      ));
    }
  }

  @override
  void dispose() {
    _generation?.cancel();
    _modelDir.dispose();
    _prompt.dispose();
    super.dispose();
  }

  Future<void> _runDoctor() async {
    final llm = await _llm.doctor();
    final vlm = await OpenVinoVlmService().doctor();
    setState(() {
      _output
        ..clear()
        ..write('LLM $llm\n\nVLM $vlm');
    });
  }

  Future<void> _load() async {
    final dir = _modelDir.text.trim();
    if (dir.isEmpty) {
      setState(() => _status = 'Enter a model directory first.');
      return;
    }
    setState(() {
      _busy = true;
      _status = 'Loading $dir …';
    });
    final watch = Stopwatch()..start();
    try {
      await _llm.loadModel(dir);
      setState(() =>
          _status = 'Ready on ${_llm.activeDevice} in ${watch.elapsed}.');
    } on OpenVinoGenAiException catch (e) {
      setState(() => _status = 'Load failed: $e');
    } finally {
      setState(() => _busy = false);
    }
  }

  Future<void> _unload() async {
    setState(() {
      _busy = true;
      _status = 'Unloading…';
    });
    await _llm.unloadModel();
    setState(() {
      _busy = false;
      _status = 'No model loaded.';
    });
  }

  void _generate() {
    _output.clear();
    setState(() => _busy = true);
    final stream = _llm.generateChatStream([
      const LlmMessage.system('You are a concise, helpful assistant.'),
      LlmMessage.user(_prompt.text),
    ]);
    _generation = stream.listen(
      (token) => setState(() => _output.write(token)),
      onError: (Object e) => setState(() {
        _status = 'Generation failed: $e';
        _busy = false;
        _generation = null;
      }),
      onDone: () => setState(() {
        _busy = false;
        _status = 'Done. (${_llm.activeDevice})';
        _generation = null;
      }),
    );
  }

  void _cancel() {
    _generation?.cancel();
    _generation = null;
    setState(() {
      _busy = false;
      _status = 'Cancelled.';
    });
  }

  @override
  Widget build(BuildContext context) => Scaffold(
        appBar: AppBar(title: const Text('openvino_genai example')),
        body: Padding(
          padding: const EdgeInsets.all(16),
          child: Column(
            crossAxisAlignment: CrossAxisAlignment.stretch,
            children: [
              TextField(
                controller: _modelDir,
                decoration: const InputDecoration(
                  labelText: 'OpenVINO IR model directory '
                      '(contains openvino_model.xml)',
                  border: OutlineInputBorder(),
                ),
              ),
              const SizedBox(height: 8),
              Wrap(
                spacing: 8,
                children: [
                  FilledButton(
                    onPressed: _busy ? null : _load,
                    child: const Text('Load model'),
                  ),
                  OutlinedButton(
                    onPressed: _busy ? null : _unload,
                    child: const Text('Unload'),
                  ),
                  OutlinedButton(
                    onPressed: _runDoctor,
                    child: const Text('doctor()'),
                  ),
                ],
              ),
              const SizedBox(height: 8),
              Text(_status, style: Theme.of(context).textTheme.bodySmall),
              const Divider(height: 24),
              TextField(
                controller: _prompt,
                decoration: const InputDecoration(
                  labelText: 'Prompt',
                  border: OutlineInputBorder(),
                ),
              ),
              const SizedBox(height: 8),
              Wrap(
                spacing: 8,
                children: [
                  FilledButton(
                    onPressed: _busy || !_llm.isRunning ? null : _generate,
                    child: const Text('Generate'),
                  ),
                  OutlinedButton(
                    onPressed: _generation == null ? null : _cancel,
                    child: const Text('Cancel'),
                  ),
                ],
              ),
              const SizedBox(height: 12),
              Expanded(
                child: Container(
                  width: double.infinity,
                  padding: const EdgeInsets.all(12),
                  decoration: BoxDecoration(
                    color: Theme.of(context).colorScheme.surfaceContainerLow,
                    borderRadius: BorderRadius.circular(8),
                  ),
                  child: SingleChildScrollView(
                    child: SelectableText(
                      _output.toString(),
                      style: const TextStyle(fontFamily: 'monospace'),
                    ),
                  ),
                ),
              ),
            ],
          ),
        ),
      );
}