vision_flow 0.0.4

A Flutter plugin for real-time vision tasks, Hands, Face, Pose estimation, and Video Classification. Built on top of MediaPipe, powered by ExecuTorch and TensorFlow Lite.

example/lib/main.dart

import 'dart:async';

import 'package:camera/camera.dart';
import 'package:file_picker/file_picker.dart';
import 'package:flutter/material.dart';
import 'package:vision_flow/vision_flow.dart';

late List<CameraDescription> cameras;

Future<void> main() async {
  WidgetsFlutterBinding.ensureInitialized();
  cameras = await availableCameras();
  runApp(const MyApp());
}

class MyApp extends StatelessWidget {
  const MyApp({super.key});

  @override
  Widget build(BuildContext context) {
    return const MaterialApp(
      title: 'VisionFlow Example',
      home: VisionFlowDemo(),
    );
  }
}

class VisionFlowDemo extends StatefulWidget {
  const VisionFlowDemo({super.key});

  @override
  State<VisionFlowDemo> createState() => _VisionFlowDemoState();
}

class _VisionFlowDemoState extends State<VisionFlowDemo> {
  CameraController? _cameraController;
  StreamSubscription? _predictionSubscription;

  String _latestPrediction = 'No Prediction';
  String _modelStatus = 'No model loaded';
  bool _isModelLoaded = false;
  bool _isProcessing = false;
  bool _isCameraRunning = false;
  int _frameCounter = 0;

  @override
  void initState() {
    super.initState();
    _initCamera();
  }

  // ─────────────────────────────────────────────
  // Camera
  // ─────────────────────────────────────────────

  Future<void> _initCamera() async {
    _cameraController = CameraController(
      cameras.first,
      ResolutionPreset.medium,
      enableAudio: false,
      imageFormatGroup: ImageFormatGroup.yuv420,
    );

    await _cameraController!.initialize();
    if (mounted) setState(() {});
  }

  void _startImageStream() {
    if (_isCameraRunning) return;
    _isCameraRunning = true;

    _cameraController!.startImageStream((CameraImage image) async {
      if (_isProcessing || !_isModelLoaded) return;

      _frameCounter++;
      if (_frameCounter % 3 != 0) return; // throttle to every 3rd frame

      _isProcessing = true;
      try {
        await VisionFlow.processFrame(
          y: image.planes[0].bytes,
          u: image.planes[1].bytes,
          v: image.planes[2].bytes,
          width: image.width,
          height: image.height,
          yRowStride: image.planes[0].bytesPerRow,
          uvRowStride: image.planes[1].bytesPerRow,
          uvPixelStride: image.planes[1].bytesPerPixel!,
        );
      } catch (e) {
        debugPrint('Frame error: $e');
      } finally {
        _isProcessing = false;
      }
    });
  }

  // ─────────────────────────────────────────────
  // Model loading helpers
  // ─────────────────────────────────────────────

  Future<void> _configureAndListen() async {
    await VisionFlow.configure(
      hands: true,
      face: true,
      pose: false,
      sequenceLength: 30,
    );

    _predictionSubscription?.cancel();
    _predictionSubscription = VisionFlow.predictions.listen((result) {
      if (!mounted) return;
      setState(() => _latestPrediction = result.label);
    });

    setState(() => _isModelLoaded = true);
    _startImageStream();
  }

  /// Option A – Load model bundled as a Flutter asset
  Future<void> _loadFromAssets() async {
    try {
      setState(() => _modelStatus = 'Loading from assets...');

      await VisionFlow.loadModel(
        path: 'assets/models/asl_sign_model.pte', // ExecuTorch format
        backend: VisionFlowModelType.pytorch,
        isAsset: true,
      );

      await _configureAndListen();
      setState(() => _modelStatus = 'Loaded from assets ✓');
    } catch (e) {
      setState(() => _modelStatus = 'Asset load failed: $e');
    }
  }

  /// Option B – Pick a model file from device storage
  Future<void> _loadFromFilePicker() async {
    try {
      final result = await FilePicker.pickFiles(
        type: FileType.custom,
        allowedExtensions: ['pte', 'tflite'], // .pte = ExecuTorch, .tflite = TFLite
        dialogTitle: 'Select a model file',
      );

      if (result == null || result.files.single.path == null) return;

      final filePath = result.files.single.path!;
      final fileName = result.files.single.name;
      setState(() => _modelStatus = 'Loading $fileName...');

      // Determine backend from file extension
      final backend = filePath.endsWith('.pte')
          ? VisionFlowModelType.pytorch
          : VisionFlowModelType.tflite;

      await VisionFlow.loadModel(
        path: filePath,
        backend: backend,
        isAsset: false, // <── absolute device path
      );

      await _configureAndListen();
      setState(() => _modelStatus = 'Loaded: $fileName ✓');
    } catch (e) {
      setState(() => _modelStatus = 'File load failed: $e');
    }
  }

  // ─────────────────────────────────────────────
  // Cleanup
  // ─────────────────────────────────────────────

  @override
  void dispose() {
    _predictionSubscription?.cancel();
    _cameraController?.dispose();
    VisionFlow.dispose();
    super.dispose();
  }

  // ─────────────────────────────────────────────
  // UI
  // ─────────────────────────────────────────────

  @override
  Widget build(BuildContext context) {
    final cameraReady =
        _cameraController != null && _cameraController!.value.isInitialized;

    return Scaffold(
      backgroundColor: const Color(0xFF0D0D1A),
      appBar: AppBar(
        backgroundColor: const Color(0xFF141428),
        title: const Text(
          'VisionFlow',
          style: TextStyle(color: Colors.white, fontWeight: FontWeight.bold),
        ),
        centerTitle: true,
      ),
      body: Column(
        children: [
          // ── Camera preview ──────────────────────────
          Expanded(
            child: cameraReady
                ? ClipRRect(
                    borderRadius: const BorderRadius.vertical(
                      bottom: Radius.circular(24),
                    ),
                    child: CameraPreview(_cameraController!),
                  )
                : const Center(
                    child: CircularProgressIndicator(color: Colors.cyanAccent),
                  ),
          ),

          const SizedBox(height: 16),

          // ── Prediction result ───────────────────────
          Container(
            margin: const EdgeInsets.symmetric(horizontal: 24),
            padding: const EdgeInsets.symmetric(vertical: 14, horizontal: 20),
            decoration: BoxDecoration(
              color: const Color(0xFF1C1C35),
              borderRadius: BorderRadius.circular(16),
              border: Border.all(color: Colors.cyanAccent.withOpacity(0.3)),
            ),
            child: Row(
              children: [
                const Icon(Icons.spatial_tracking, color: Colors.cyanAccent),
                const SizedBox(width: 12),
                Expanded(
                  child: Text(
                    _latestPrediction,
                    style: const TextStyle(
                      color: Colors.white,
                      fontSize: 20,
                      fontWeight: FontWeight.bold,
                    ),
                  ),
                ),
              ],
            ),
          ),

          const SizedBox(height: 12),

          // ── Model status ────────────────────────────
          Padding(
            padding: const EdgeInsets.symmetric(horizontal: 24),
            child: Text(
              _modelStatus,
              style: TextStyle(
                color: _isModelLoaded ? Colors.greenAccent : Colors.grey[500],
                fontSize: 13,
              ),
              textAlign: TextAlign.center,
            ),
          ),

          const SizedBox(height: 16),

          // ── Load buttons ────────────────────────────
          Padding(
            padding: const EdgeInsets.symmetric(horizontal: 24),
            child: Row(
              children: [
                // Load from assets
                Expanded(
                  child: ElevatedButton.icon(
                    onPressed: _loadFromAssets,
                    icon: const Icon(Icons.inventory_2_outlined),
                    label: const Text('Load from Assets'),
                    style: ElevatedButton.styleFrom(
                      backgroundColor: const Color(0xFF0077FF),
                      foregroundColor: Colors.white,
                      padding: const EdgeInsets.symmetric(vertical: 14),
                      shape: RoundedRectangleBorder(
                        borderRadius: BorderRadius.circular(12),
                      ),
                    ),
                  ),
                ),

                const SizedBox(width: 12),

                // Pick from file system
                Expanded(
                  child: ElevatedButton.icon(
                    onPressed: _loadFromFilePicker,
                    icon: const Icon(Icons.folder_open_outlined),
                    label: const Text('Browse File'),
                    style: ElevatedButton.styleFrom(
                      backgroundColor: const Color(0xFF7C3AED),
                      foregroundColor: Colors.white,
                      padding: const EdgeInsets.symmetric(vertical: 14),
                      shape: RoundedRectangleBorder(
                        borderRadius: BorderRadius.circular(12),
                      ),
                    ),
                  ),
                ),
              ],
            ),
          ),

          const SizedBox(height: 28),
        ],
      ),
    );
  }
}