flutter_edge_ai_llama 0.0.1

Cross-platform GGUF inference for Flutter Edge AI, powered by llamadart with tools, thinking, multimodal input and native LoRA.

example/lib/main.dart

import 'dart:async';

import 'package:file_selector/file_selector.dart';
import 'package:flutter/material.dart';
import 'package:flutter/foundation.dart';
import 'package:flutter_edge_ai/flutter_edge_ai.dart' as edge;
import 'package:flutter_edge_ai_llama/flutter_edge_ai_llama.dart';
import 'package:llamadart/llamadart.dart' as llama;

import 'model_import.dart';

Future<void> main() async {
  WidgetsFlutterBinding.ensureInitialized();
  edge.FlutterEdgeAi.logLevel = edge.EdgeAiLogLevel.none;
  llama.LlamaLogging.configure(level: llama.LlamaLogLevel.error);
  await edge.FlutterEdgeAi.initialize(
    inferenceEngines: const [LlamaEngineProvider()],
  );
  runApp(
    MaterialApp(
      title: 'GGUF Lab',
      debugShowCheckedModeBanner: false,
      theme: ThemeData(
        brightness: Brightness.dark,
        colorSchemeSeed: const Color(0xFFB6E78C),
        scaffoldBackgroundColor: const Color(0xFF121817),
        useMaterial3: true,
      ),
      home: const _ChatPage(),
    ),
  );
}

class _ChatPage extends StatefulWidget {
  const _ChatPage();
  @override
  State<_ChatPage> createState() => _ChatPageState();
}

class _ChatPageState extends State<_ChatPage> with WidgetsBindingObserver {
  final _path = TextEditingController(
    text: const String.fromEnvironment('GGUF_MODEL_PATH'),
  );
  final _prompt = TextEditingController();
  final _scroll = ScrollController();
  final List<_Turn> _turns = [];
  edge.InferenceModel? _model;
  edge.InferenceChat? _chat;
  bool _loading = false;
  bool _generating = false;
  bool _stopped = false;
  Timer? _renderTimer;
  String _status = kIsWeb || defaultTargetPlatform == TargetPlatform.android
      ? 'Enter a GGUF URL to begin.'
      : 'Load a local GGUF model to begin.';
  String _backend = 'Not loaded';

  @override
  void initState() {
    super.initState();
    WidgetsBinding.instance.addObserver(this);
    unawaited(_resumeModel());
  }

  Future<void> _resumeModel() async {
    if (edge.FlutterEdgeAiPlugin.instance.modelManager.activeInferenceModel ==
        null) {
      return;
    }
    setState(() {
      _loading = true;
      _status = 'Restoring the installed model…';
    });
    try {
      final loaded = await edge.FlutterEdgeAi.getActiveModel(maxTokens: 2048);
      if (!mounted) {
        await loaded.close();
        return;
      }
      _model = loaded;
      final restoredSource = edge
          .FlutterEdgeAiPlugin
          .instance
          .modelManager
          .activeInferenceModel!
          .files
          .first
          .source;
      _path.text = restoredModelInput(
        restoredSource,
        urlsOnly: kIsWeb || defaultTargetPlatform == TargetPlatform.android,
      );
      if (!kIsWeb && restoredSource is edge.FileSource) {
        await pruneImportedModels(restoredSource.path);
      }
      await _newChat();
      if (mounted) {
        setState(() {
          _backend = loaded.activeBackend?.name.toUpperCase() ?? 'UNKNOWN';
          _status = 'Installed model restored. Start a new conversation.';
        });
      }
    } catch (error) {
      if (mounted) {
        setState(() => _status = 'Could not restore the model: $error');
      }
    } finally {
      if (mounted) setState(() => _loading = false);
    }
  }

  @override
  void didChangeAppLifecycleState(AppLifecycleState state) {
    if (state == AppLifecycleState.paused ||
        state == AppLifecycleState.hidden) {
      unawaited(_stop());
    }
  }

  Future<void> _chooseModel() async {
    if (_loading || _generating) return;
    if (kIsWeb || defaultTargetPlatform == TargetPlatform.android) {
      await _load();
      return;
    }
    setState(() => _loading = true);
    try {
      final file = await openFile(
        acceptedTypeGroups: [
          XTypeGroup(
            label: 'GGUF models',
            extensions: const ['gguf'],
            uniformTypeIdentifiers: defaultTargetPlatform == TargetPlatform.iOS
                ? const ['public.data']
                : null,
          ),
        ],
      );
      if (file == null || !mounted) return;
      setState(() => _status = 'Importing model into app storage…');
      final imported = await importModel(
        file.path,
        cleanupPickerInbox: defaultTargetPlatform == TargetPlatform.iOS,
      );
      if (!mounted) return;
      _path.text = imported;
      await _load();
    } catch (error) {
      if (mounted) setState(() => _status = error.toString());
    } finally {
      if (mounted) setState(() => _loading = false);
    }
  }

  void _clearTurns() {
    for (final turn in _turns) {
      turn.dispose();
    }
    _turns.clear();
  }

  Future<void> _load() async {
    if (_path.text.trim().isEmpty) return;
    final network = kIsWeb || defaultTargetPlatform == TargetPlatform.android;
    final selected = _path.text.trim();
    if (network) {
      try {
        validateModelUrl(selected);
      } on ArgumentError catch (error) {
        setState(() => _status = error.message.toString());
        return;
      }
    }
    final previous =
        edge.FlutterEdgeAiPlugin.instance.modelManager.activeInferenceModel;
    setState(() {
      _loading = true;
      _chat = null;
      _status = 'Loading model…';
      _backend = 'Not loaded';
      _clearTurns();
    });
    try {
      await _model?.close();
      _model = null;
      final installation = edge.FlutterEdgeAi.installModel(
        modelType: edge.ModelType.general,
        fileType: edge.ModelFileType.gguf,
      );
      if (network) {
        await installation.fromNetwork(selected).install();
      } else {
        await installation.fromFile(selected).install();
      }
      final loaded = await edge.FlutterEdgeAi.getActiveModel(maxTokens: 2048);
      if (!mounted) {
        await loaded.close();
        return;
      }
      _model = loaded;
      if (!kIsWeb && defaultTargetPlatform != TargetPlatform.android) {
        await pruneImportedModels(_path.text.trim());
      }
      await _newChat();
      if (mounted) {
        setState(() {
          _backend = _model!.activeBackend?.name.toUpperCase() ?? 'UNKNOWN';
          _status =
              'Ready · ${_model!.maxTokens} context tokens · 128 output tokens';
        });
      }
      final active =
          edge.FlutterEdgeAiPlugin.instance.modelManager.activeInferenceModel;
      if ((kIsWeb || defaultTargetPlatform == TargetPlatform.android) &&
          shouldUninstallPreviousModel(previous, active)) {
        try {
          await edge.FlutterEdgeAi.uninstallModel(
            previous!.files.first.filename,
          );
        } catch (error) {
          // A cleanup failure must not discard the newly loaded model.
          if (mounted) {
            setState(
              () => _status =
                  'Ready. Could not remove the previous download: $error',
            );
          }
        }
      }
    } catch (error) {
      if (mounted) setState(() => _status = error.toString());
    } finally {
      if (mounted) setState(() => _loading = false);
    }
  }

  Future<void> _newChat() async {
    final chat = await _model!.createChat(
      systemInstruction: 'You are a helpful assistant. Keep answers concise.',
      modelType: edge.ModelType.general,
      temperature: .7,
      topK: 40,
      topP: .9,
      maxOutputTokens: 128,
    );
    if (!mounted) {
      await chat.close();
      return;
    }
    _chat = chat;
    if (mounted) {
      setState(() {
        _clearTurns();
        _status = 'New conversation ready.';
      });
    }
  }

  Future<void> _send() async {
    final input = _prompt.text.trim();
    final chat = _chat;
    if (input.isEmpty || chat == null || _generating || _loading) return;
    final reply = _Turn(false, '');
    setState(() {
      _generating = true;
      _stopped = false;
      _status = 'Generating locally…';
      _turns.addAll([_Turn(true, input), reply]);
      _prompt.clear();
    });
    var committed = false;
    try {
      await chat.addQueryChunk(edge.Message.text(text: input, isUser: true));
      await for (final response in chat.generateChatResponseAsync()) {
        if (!mounted) break;
        if (response is edge.TextResponse) {
          reply.append(response.token);
          _renderTimer ??= Timer(const Duration(milliseconds: 33), () {
            _renderTimer = null;
            if (!mounted) return;
            final follow =
                !_scroll.hasClients || _scroll.position.extentAfter < 120;
            reply.flush();
            if (follow) {
              WidgetsBinding.instance.addPostFrameCallback((_) {
                if (mounted && _scroll.hasClients) {
                  _scroll.jumpTo(_scroll.position.maxScrollExtent);
                }
              });
            }
          });
        }
      }
      committed = !_stopped;
      final metrics = chat.session.getSessionMetrics();
      if (mounted) {
        setState(() {
          _status = _stopped
              ? 'Stopped. The interrupted turn was discarded; you can continue.'
              : '${metrics.inputTokens} input · ${metrics.outputTokens > 0 ? '${metrics.outputTokens} output tokens' : 'output count unavailable'}';
        });
      }
    } catch (error) {
      if (mounted) {
        setState(() {
          _status = '$error The failed turn was discarded.';
        });
      }
    } finally {
      _renderTimer?.cancel();
      _renderTimer = null;
      if (mounted) {
        reply.flush();
        setState(() {
          if (!committed) {
            _turns.remove(reply);
            reply.dispose();
            if (_turns.isNotEmpty && _turns.last.isUser) {
              _turns.removeLast().dispose();
            }
          }
          _generating = false;
        });
      }
    }
  }

  Future<void> _stop() async {
    _stopped = true;
    await _chat?.session.stopGeneration();
  }

  @override
  void dispose() {
    WidgetsBinding.instance.removeObserver(this);
    _renderTimer?.cancel();
    _clearTurns();
    unawaited(_model?.close());
    _path.dispose();
    _prompt.dispose();
    _scroll.dispose();
    super.dispose();
  }

  @override
  Widget build(BuildContext context) => Scaffold(
    body: SafeArea(
      child: Center(
        child: ConstrainedBox(
          constraints: const BoxConstraints(maxWidth: 960),
          child: Padding(
            padding: EdgeInsets.all(
              MediaQuery.sizeOf(context).width < 520 ? 12 : 28,
            ),
            child: Column(
              crossAxisAlignment: CrossAxisAlignment.start,
              children: [
                if (MediaQuery.viewInsetsOf(context).bottom == 0) ...[
                  Wrap(
                    spacing: 16,
                    runSpacing: 8,
                    crossAxisAlignment: WrapCrossAlignment.center,
                    children: [
                      const Row(
                        mainAxisSize: MainAxisSize.min,
                        children: [
                          Icon(
                            Icons.memory,
                            color: Color(0xFFB6E78C),
                            size: 32,
                          ),
                          SizedBox(width: 12),
                          Text(
                            'GGUF Lab',
                            style: TextStyle(
                              fontSize: 30,
                              fontWeight: FontWeight.w700,
                            ),
                          ),
                        ],
                      ),
                      Chip(label: Text(_backend)),
                    ],
                  ),
                  const SizedBox(height: 8),
                  const Text(
                    'Flutter Edge AI × llamadart',
                    style: TextStyle(color: Color(0xFFA5B9B0), fontSize: 16),
                  ),
                  const SizedBox(height: 24),
                  LayoutBuilder(
                    builder: (context, constraints) {
                      final field = TextField(
                        controller: _path,
                        readOnly:
                            !kIsWeb &&
                            defaultTargetPlatform != TargetPlatform.android,
                        decoration: InputDecoration(
                          labelText:
                              kIsWeb ||
                                  defaultTargetPlatform ==
                                      TargetPlatform.android
                              ? 'GGUF model URL'
                              : 'Local GGUF model',
                          border: OutlineInputBorder(),
                        ),
                      );
                      final button = FilledButton(
                        onPressed: _loading || _generating
                            ? null
                            : _chooseModel,
                        child: Text(
                          _loading
                              ? 'Loading…'
                              : kIsWeb ||
                                    defaultTargetPlatform ==
                                        TargetPlatform.android
                              ? 'Load URL'
                              : 'Choose model',
                        ),
                      );
                      if (constraints.maxWidth < 520) {
                        return Column(
                          crossAxisAlignment: CrossAxisAlignment.stretch,
                          children: [field, const SizedBox(height: 8), button],
                        );
                      }
                      return Row(
                        children: [
                          Expanded(child: field),
                          const SizedBox(width: 12),
                          button,
                        ],
                      );
                    },
                  ),
                  const SizedBox(height: 12),
                  Row(
                    children: [
                      Expanded(
                        child: Text(
                          _status,
                          style: const TextStyle(color: Color(0xFFA5B9B0)),
                        ),
                      ),
                      TextButton(
                        onPressed: _model == null || _generating || _loading
                            ? null
                            : _newChat,
                        child: const Text('New chat'),
                      ),
                    ],
                  ),
                ],
                const Divider(height: 28),
                Expanded(
                  child: _turns.isEmpty
                      ? const Center(
                          child: Text(
                            'Your model. Your device.\nStart a conversation below.',
                            textAlign: TextAlign.center,
                            style: TextStyle(
                              fontSize: 22,
                              color: Color(0xFFA5B9B0),
                            ),
                          ),
                        )
                      : ListView.builder(
                          controller: _scroll,
                          itemCount: _turns.length,
                          itemBuilder: (context, index) {
                            final turn = _turns[index];
                            return Padding(
                              padding: const EdgeInsets.only(bottom: 16),
                              child: DecoratedBox(
                                decoration: BoxDecoration(
                                  color: turn.isUser
                                      ? const Color(0xFF253A31)
                                      : const Color(0xFF1B2621),
                                  borderRadius: BorderRadius.circular(14),
                                ),
                                child: Padding(
                                  padding: const EdgeInsets.all(18),
                                  child: Column(
                                    crossAxisAlignment:
                                        CrossAxisAlignment.start,
                                    children: [
                                      Text(
                                        turn.isUser ? 'YOU' : 'LOCAL MODEL',
                                        style: const TextStyle(
                                          fontSize: 11,
                                          letterSpacing: 1.5,
                                          color: Color(0xFFB6E78C),
                                        ),
                                      ),
                                      const SizedBox(height: 8),
                                      ValueListenableBuilder<String>(
                                        valueListenable: turn.text,
                                        builder: (_, text, _) => SelectableText(
                                          text.isEmpty ? '…' : text,
                                          style: const TextStyle(
                                            fontSize: 16,
                                            height: 1.5,
                                          ),
                                        ),
                                      ),
                                    ],
                                  ),
                                ),
                              ),
                            );
                          },
                        ),
                ),
                const SizedBox(height: 16),
                Row(
                  children: [
                    Expanded(
                      child: TextField(
                        controller: _prompt,
                        enabled: _chat != null && !_generating,
                        decoration: const InputDecoration(
                          hintText: 'Ask your local model…',
                          border: OutlineInputBorder(),
                        ),
                        onSubmitted: (_) => _send(),
                      ),
                    ),
                    const SizedBox(width: 12),
                    FilledButton(
                      onPressed: _generating
                          ? _stop
                          : _chat == null
                          ? null
                          : _send,
                      child: Text(_generating ? 'Stop' : 'Send'),
                    ),
                  ],
                ),
              ],
            ),
          ),
        ),
      ),
    ),
  );
}

class _Turn {
  _Turn(this.isUser, String initial) : text = ValueNotifier(initial);
  final bool isUser;
  final ValueNotifier<String> text;
  final StringBuffer _buffer = StringBuffer();

  void append(String token) => _buffer.write(token);
  void flush() => text.value = _buffer.toString();
  void dispose() => text.dispose();
}

/// Rejects names and incomplete URLs before replacing the loaded model.
void validateModelUrl(String value) {
  final uri = Uri.tryParse(value);
  if (uri == null ||
      !(uri.scheme == 'https' ||
          uri.scheme == 'http' &&
              ['localhost', '127.0.0.1', '::1'].contains(uri.host)) ||
      uri.host.isEmpty ||
      uri.pathSegments.isEmpty ||
      uri.pathSegments.last.isEmpty ||
      !kIsWeb &&
          defaultTargetPlatform == TargetPlatform.android &&
          uri.scheme != 'https') {
    throw ArgumentError(
      'Enter a complete HTTPS GGUF model URL (HTTP is allowed only for local testing outside Android).',
    );
  }
}

/// Keeps a rehydrated local cache path out of the URL-only replacement field.
@visibleForTesting
String restoredModelInput(edge.ModelSource source, {required bool urlsOnly}) =>
    switch (source) {
      edge.NetworkSource(:final url) => url,
      edge.FileSource(:final path) => urlsOnly ? '' : path,
      _ => '',
    };

/// Core owns deletion policy: uninstall removes managed downloads, while
/// external fromFile installs keep their bytes even after restart hydration.
@visibleForTesting
bool shouldUninstallPreviousModel(
  edge.ModelSpec? previous,
  edge.ModelSpec? active,
) =>
    previous != null &&
    active != null &&
    previous.files.first.filename != active.files.first.filename;