openvino_genai 0.1.0

On-device LLM inference for Flutter via Intel OpenVINO GenAI: streaming chat completion on CPU/GPU with hardened load and fallback behaviour. Windows x64; unofficial.

example/lib/main.dart

import 'dart:async';
import 'dart:io';

import 'package:flutter/material.dart';
import 'package:openvino_genai/openvino_genai.dart';

void main() {
  // Optional device-ladder override, e.g. "NPU,GPU,CPU" (default GPU,CPU).
  final devices = Platform.environment['OPENVINO_GENAI_EXAMPLE_DEVICES'];
  if (devices != null && devices.trim().isNotEmpty) {
    OpenVinoGenAiService.devicePreference = [
      for (final d in devices.split(','))
        if (d.trim().isNotEmpty) d.trim(),
    ];
  }
  // Doctor mode: bind the bridge, print diagnostics, exit 0/1. Needs no
  // model and no GPU — CI uses it to prove the bundled DLL set loads and
  // all 12 exports bind on a clean machine.
  if (Platform.environment['OPENVINO_GENAI_EXAMPLE_DOCTOR'] == '1' &&
      !Platform.environment.containsKey('FLUTTER_TEST')) {
    unawaited(_runDoctorMode());
    return;
  }
  runApp(const ExampleApp());
}

Future<void> _runDoctorMode() async {
  final report = await OpenVinoGenAiService().doctor();
  stdout.writeln('[doctor] $report');
  // CPU must appear in the enumeration: that proves openvino.dll and
  // openvino_c.dll actually loaded and answered, not just that the bridge
  // DLL bound.
  final ok = report.dllFound &&
      report.bridgeBound &&
      (report.availableDevices?.contains('CPU') ?? false);
  stdout.writeln(ok ? '[doctor] PASS' : '[doctor] FAIL');
  exit(ok ? 0 : 1);
}

/// Self-test lane for CI / smoke testing: when
/// `OPENVINO_GENAI_EXAMPLE_MODEL` is set to a model directory, the app loads
/// it, runs one short greedy generation, prints the tokens to stdout and
/// exits — 0 on success, 1 on failure.
Future<void> runAutoTest(String modelDir) async {
  final llm = OpenVinoGenAiService();
  try {
    stdout.writeln('[autotest] doctor:\n${await llm.doctor()}');
    stdout.writeln('[autotest] loading $modelDir');
    final watch = Stopwatch()..start();
    await llm.loadModel(modelDir);
    stdout.writeln(
        '[autotest] loaded on ${llm.activeDevice} in ${watch.elapsed}');
    final reply = await llm.generateText(
      [
        const LlmMessage.system('You are a terse assistant.'),
        const LlmMessage.user('Reply with the single word: OK'),
      ],
      params: LlmGenerationParams.greedy.copyWith(maxTokens: 16),
    );
    stdout.writeln('[autotest] reply: $reply');
    await llm.unloadModel();
    stdout.writeln('[autotest] PASS');
    exit(0);
  } on Object catch (e) {
    stdout.writeln('[autotest] FAIL: $e');
    exit(1);
  }
}

/// Minimal demo: pick an OpenVINO IR model directory, load it, chat.
class ExampleApp extends StatelessWidget {
  const ExampleApp({super.key});

  @override
  Widget build(BuildContext context) => MaterialApp(
        title: 'openvino_genai example',
        theme: ThemeData(colorSchemeSeed: Colors.indigo, useMaterial3: true),
        home: const DemoPage(),
      );
}

/// One page: model controls on top, streamed chat output below.
class DemoPage extends StatefulWidget {
  const DemoPage({super.key});

  @override
  State<DemoPage> createState() => _DemoPageState();
}

class _DemoPageState extends State<DemoPage> {
  final _llm = OpenVinoGenAiService();
  final _modelDir = TextEditingController();
  final _prompt = TextEditingController(text: 'Why is the sky blue?');
  final _output = StringBuffer();

  String _status = 'No model loaded.';
  bool _busy = false;
  StreamSubscription<String>? _generation;

  @override
  void initState() {
    super.initState();
    final autoModel = Platform.environment['OPENVINO_GENAI_EXAMPLE_MODEL'];
    // FLUTTER_TEST guard: with the env var exported in the same shell,
    // `flutter test` would otherwise trigger real FFI loads and exit() —
    // hard-killing the test harness.
    if (autoModel != null &&
        autoModel.isNotEmpty &&
        !Platform.environment.containsKey('FLUTTER_TEST')) {
      _modelDir.text = autoModel;
      unawaited(runAutoTest(autoModel));
    }
  }

  @override
  void dispose() {
    _generation?.cancel();
    _modelDir.dispose();
    _prompt.dispose();
    super.dispose();
  }

  Future<void> _runDoctor() async {
    final report = await _llm.doctor();
    setState(() {
      _output
        ..clear()
        ..write(report);
    });
  }

  Future<void> _load() async {
    final dir = _modelDir.text.trim();
    if (dir.isEmpty) {
      setState(() => _status = 'Enter a model directory first.');
      return;
    }
    setState(() {
      _busy = true;
      _status = 'Loading $dir …';
    });
    final watch = Stopwatch()..start();
    try {
      await _llm.loadModel(dir);
      setState(() =>
          _status = 'Ready on ${_llm.activeDevice} in ${watch.elapsed}.');
    } on OpenVinoGenAiException catch (e) {
      setState(() => _status = 'Load failed: $e');
    } finally {
      setState(() => _busy = false);
    }
  }

  Future<void> _unload() async {
    setState(() {
      _busy = true;
      _status = 'Unloading…';
    });
    await _llm.unloadModel();
    setState(() {
      _busy = false;
      _status = 'No model loaded.';
    });
  }

  void _generate() {
    _output.clear();
    setState(() => _busy = true);
    final stream = _llm.generateChatStream([
      const LlmMessage.system('You are a concise, helpful assistant.'),
      LlmMessage.user(_prompt.text),
    ]);
    _generation = stream.listen(
      (token) => setState(() => _output.write(token)),
      onError: (Object e) => setState(() {
        _status = 'Generation failed: $e';
        _busy = false;
        _generation = null;
      }),
      onDone: () => setState(() {
        _busy = false;
        _status = 'Done. (${_llm.activeDevice})';
        _generation = null;
      }),
    );
  }

  void _cancel() {
    _generation?.cancel();
    _generation = null;
    setState(() {
      _busy = false;
      _status = 'Cancelled.';
    });
  }

  @override
  Widget build(BuildContext context) => Scaffold(
        appBar: AppBar(title: const Text('openvino_genai example')),
        body: Padding(
          padding: const EdgeInsets.all(16),
          child: Column(
            crossAxisAlignment: CrossAxisAlignment.stretch,
            children: [
              TextField(
                controller: _modelDir,
                decoration: const InputDecoration(
                  labelText: 'OpenVINO IR model directory '
                      '(contains openvino_model.xml)',
                  border: OutlineInputBorder(),
                ),
              ),
              const SizedBox(height: 8),
              Wrap(
                spacing: 8,
                children: [
                  FilledButton(
                    onPressed: _busy ? null : _load,
                    child: const Text('Load model'),
                  ),
                  OutlinedButton(
                    onPressed: _busy ? null : _unload,
                    child: const Text('Unload'),
                  ),
                  OutlinedButton(
                    onPressed: _runDoctor,
                    child: const Text('doctor()'),
                  ),
                ],
              ),
              const SizedBox(height: 8),
              Text(_status, style: Theme.of(context).textTheme.bodySmall),
              const Divider(height: 24),
              TextField(
                controller: _prompt,
                decoration: const InputDecoration(
                  labelText: 'Prompt',
                  border: OutlineInputBorder(),
                ),
              ),
              const SizedBox(height: 8),
              Wrap(
                spacing: 8,
                children: [
                  FilledButton(
                    onPressed: _busy || !_llm.isRunning ? null : _generate,
                    child: const Text('Generate'),
                  ),
                  OutlinedButton(
                    onPressed: _generation == null ? null : _cancel,
                    child: const Text('Cancel'),
                  ),
                ],
              ),
              const SizedBox(height: 12),
              Expanded(
                child: Container(
                  width: double.infinity,
                  padding: const EdgeInsets.all(12),
                  decoration: BoxDecoration(
                    color: Theme.of(context).colorScheme.surfaceContainerLow,
                    borderRadius: BorderRadius.circular(8),
                  ),
                  child: SingleChildScrollView(
                    child: SelectableText(
                      _output.toString(),
                      style: const TextStyle(fontFamily: 'monospace'),
                    ),
                  ),
                ),
              ),
            ],
          ),
        ),
      );
}