vision_flow 0.0.2+1

A Flutter plugin for real-time vision tasks, Hands, Face, Pose estimation, and Video Classification. Built on top of MediaPipe, powered by PyTorch and TensorFlow Lite.

example/lib/main.dart

import 'dart:async';

import 'package:camera/camera.dart';
import 'package:flutter/material.dart';
import 'package:signify_vision/vision_flow.dart';

late List<CameraDescription> cameras;

Future<void> main() async {
  WidgetsFlutterBinding.ensureInitialized();
  cameras = await availableCameras();

  runApp(const MyApp());
}

class MyApp extends StatefulWidget {
  const MyApp({super.key});

  @override
  State<MyApp> createState() => _MyAppState();
}

class _MyAppState extends State<MyApp> {
  late CameraController controller;
  StreamSubscription? predictionSubscription;
  String latestPrediction = "No Prediction";

  @override
  void initState() {
    super.initState();
    initPlugin();
    initCamera();
  }

  Future<void> initPlugin() async {
    // Note: ensure you have added a model to your assets folder and pubspec.yaml
    // For testing purposes, we comment out loadModel if you don't have a model yet.
    /*
    await VisionFlow.loadModel(
      path: "assets/model.pt",
      backend: VisionFlowModelType.pytorch,
    );
    */

    await VisionFlow.configure(
      hands: true,
      face: true,
      pose: false,
      sequenceLength: 30,
    );

    predictionSubscription = VisionFlow.predictions.listen((result) {
      print("Prediction received: ${result.label} (Index: ${result.index})");
      setState(() {
        latestPrediction = result.label;
      });
    });
  }

  Future<void> initCamera() async {
    controller = CameraController(
      cameras[0],
      ResolutionPreset.medium,
      enableAudio: false,
    );

    await controller.initialize();

    bool isProcessing = false;

    await controller.startImageStream((CameraImage image) async {
      if (isProcessing) return;

      isProcessing = true;

      try {
        await VisionFlow.processFrame(
          y: image.planes[0].bytes,
          u: image.planes[1].bytes,
          v: image.planes[2].bytes,
          width: image.width,
          height: image.height,
          yRowStride: image.planes[0].bytesPerRow,
          uvRowStride: image.planes[1].bytesPerRow,
          uvPixelStride: image.planes[1].bytesPerPixel ?? 1,
        );
      } catch (e) {
        print("Error processing frame: $e");
      } finally {
        isProcessing = false;
      }
    });

    if (mounted) {
      setState(() {});
    }
  }

  @override
  void dispose() {
    controller.dispose();
    predictionSubscription?.cancel();
    VisionFlow.dispose();
    super.dispose();
  }

  @override
  Widget build(BuildContext context) {
    if (!controller.value.isInitialized) {
      return const MaterialApp(
        home: Scaffold(body: Center(child: CircularProgressIndicator())),
      );
    }

    return MaterialApp(
      home: Scaffold(
        appBar: AppBar(title: const Text('VisionFlow Example')),
        body: Stack(
          children: [
            CameraPreview(controller),
            Positioned(
              bottom: 50,
              left: 0,
              right: 0,
              child: Center(
                child: Container(
                  padding: const EdgeInsets.symmetric(
                    vertical: 10,
                    horizontal: 20,
                  ),
                  decoration: BoxDecoration(
                    color: Colors.black54,
                    borderRadius: BorderRadius.circular(10),
                  ),
                  child: Text(
                    latestPrediction,
                    style: const TextStyle(
                      color: Colors.white,
                      fontSize: 24,
                      fontWeight: FontWeight.bold,
                    ),
                  ),
                ),
              ),
            ),
          ],
        ),
      ),
    );
  }
}