utd_video_effects_kit 0.9.0

Real-time video filters and beauty effects for LiveKit Flutter apps — LUT color grading, skin retouch, makeup, accessories, and background blur as a livekit_client TrackProcessor.

example/lib/main.dart

import 'dart:async';

import 'package:flutter/material.dart';
import 'package:livekit_client/livekit_client.dart';
import 'package:utd_video_effects_kit/utd_video_effects_kit.dart';

import 'diagnostics_page.dart';

/// Demo: open the camera with a [VideoEffectsProcessor] attached and drive the
/// whole effect surface through the kit's drop-in [VideoEffectsSheet] (beauty
/// sliders, LUT filters, makeup looks, accessories, background blur).
///
/// Run on a REAL DEVICE — simulators/emulators hit the I420/CPU fallback and the
/// MediaPipe face pass won't be representative. Platform runners (android/ ios/)
/// are generated with `flutter create .` in this directory.
void main() => runApp(const _EffectsDemoApp());

class _EffectsDemoApp extends StatelessWidget {
  const _EffectsDemoApp();

  @override
  Widget build(BuildContext context) {
    return const MaterialApp(
      debugShowCheckedModeBanner: false,
      home: _EffectsDemoPage(),
    );
  }
}

class _EffectsDemoPage extends StatefulWidget {
  const _EffectsDemoPage();

  @override
  State<_EffectsDemoPage> createState() => _EffectsDemoPageState();
}

class _EffectsDemoPageState extends State<_EffectsDemoPage> {
  final VideoEffectsProcessor _fx = VideoEffectsProcessor.create();
  LocalVideoTrack? _track;
  // Camera acquisition failure (permission denied / camera busy). Rendered with
  // a retry button instead of stranding the page on the spinner forever.
  String? _startError;
  // Live tracking-telemetry chip (long-press the preview to toggle): surfaces
  // the numbers a lag report needs — which tracking path the device is on
  // (sync/async, delegate, detect input px), the landmark age at draw, and the
  // render p95 — without digging through the Diagnostics screens.
  bool _telemetryOn = false;
  Timer? _telemetryTimer;
  String _telemetry = '';

  @override
  void initState() {
    super.initState();
    _start();
  }

  Future<void> _start() async {
    // This page is the reference integration: without the try/catch a camera
    // permission denial would surface as an unhandled zone error and leave the
    // spinner up forever.
    try {
      final track = await LocalVideoTrack.createCameraTrack(
        CameraCaptureOptions(processor: _fx),
      );
      await _fx.setEnabled(true);
      // Device-validation default: light the DEBUG_LATENCY logcat probes from
      // startup so every plain run captures the landmarker/pipeline latency
      // story — without this, the lines only appear after someone remembers
      // to open telemetry/Diagnostics (session must be attached; it is, since
      // createCameraTrack attaches before setEnabled resolves).
      await _fx.setDebugFlags({'debugLatency': true});
      if (mounted) setState(() => _track = track);
    } catch (e) {
      if (mounted) setState(() => _startError = '$e');
    }
  }

  Future<void> _toggleTelemetry() async {
    if (_telemetryOn) {
      _telemetryTimer?.cancel();
      setState(() {
        _telemetryOn = false;
        _telemetry = '';
      });
      // Deliberately leave debugLatency ON: the logcat probes are the device
      // A/B record and must survive toggling the on-screen overlay off.
      return;
    }
    setState(() => _telemetryOn = true);
    await _fx.setDebugFlags({'debugLatency': true});
    await _fx.perfStats(reset: true);
    _telemetryTimer = Timer.periodic(const Duration(seconds: 1), (_) async {
      final info = await _fx.debugInfo();
      final perf = await _fx.perfStats();
      if (!mounted || !_telemetryOn) return;
      String n(Object? v, [int digits = 1]) =>
          v is num ? v.toStringAsFixed(digits) : '—';
      setState(() {
        _telemetry = 'track ${info?['syncActive'] == true ? 'SYNC' : 'async'}'
            '${info?['syncFellBack'] == true ? ' (fell back)' : ''}'
            '  ${info?['delegate'] ?? '—'}'
            '  in ${info?['syncDetectLong'] ?? '—'}px\n'
            'inferMs ${n(info?['syncInferMs'])}'
            '  ageMs ${n(info?['landmarkAgeMs'])}'
            '  asyncLatMs ${n(info?['avgLatencyMs'])}\n'
            'p95 ${n(perf?.p95Ms)}ms  level ${_topLevel(perf)}';
      });
    });
  }

  static String _topLevel(EffectsPerfStats? p) {
    if (p == null) return '—';
    var top = 0;
    var topFrames = -1;
    for (var i = 0; i < p.levelFrames.length; i++) {
      if (p.levelFrames[i] > topFrames) {
        topFrames = p.levelFrames[i];
        top = i;
      }
    }
    return 'L$top';
  }

  @override
  void dispose() {
    _telemetryTimer?.cancel();
    // Let LiveKit's async stop (which calls processor.destroy() mid-way) finish
    // BEFORE releasing the processor's notifiers — the reverse order would race
    // destroy() against dispose() on every teardown.
    final stopped = _track?.stop();
    if (stopped != null) {
      stopped.whenComplete(_fx.dispose);
    } else {
      _fx.dispose();
    }
    super.dispose();
  }

  @override
  Widget build(BuildContext context) {
    final track = _track;
    return Scaffold(
      appBar: AppBar(
        title: const Text('Video Effects Demo'),
        actions: [
          IconButton(
            tooltip: 'Tracking telemetry',
            icon: Icon(_telemetryOn ? Icons.speed : Icons.speed_outlined),
            onPressed: _toggleTelemetry,
          ),
          IconButton(
            tooltip: 'Diagnostics',
            icon: const Icon(Icons.troubleshoot),
            onPressed: () => Navigator.of(context).push(
              MaterialPageRoute<void>(
                builder: (_) => DiagnosticsPage(processor: _fx, track: _track),
              ),
            ),
          ),
        ],
      ),
      floatingActionButton: FloatingActionButton.extended(
        icon: const Icon(Icons.auto_fix_high),
        label: const Text('Effects'),
        onPressed: () => VideoEffectsSheet.show(context, processor: _fx),
      ),
      body: Column(
        children: [
          Expanded(
            child: Stack(
              children: [
                Positioned.fill(
                  child: track == null
                      ? Center(
                          child: _startError == null
                              ? const CircularProgressIndicator()
                              : Column(
                                  mainAxisSize: MainAxisSize.min,
                                  children: [
                                    Padding(
                                      padding: const EdgeInsets.all(12),
                                      child: Text(
                                        'Camera unavailable: $_startError',
                                        style:
                                            const TextStyle(color: Colors.orange),
                                        textAlign: TextAlign.center,
                                      ),
                                    ),
                                    TextButton(
                                      onPressed: () {
                                        setState(() => _startError = null);
                                        _start();
                                      },
                                      child: const Text('Retry'),
                                    ),
                                  ],
                                ),
                        )
                      : VideoTrackRenderer(track,
                          mirrorMode: VideoViewMirrorMode.mirror),
                ),
                if (_telemetryOn && _telemetry.isNotEmpty)
                  Positioned(
                    left: 8,
                    top: 8,
                    child: Container(
                      padding: const EdgeInsets.symmetric(
                          horizontal: 8, vertical: 6),
                      decoration: BoxDecoration(
                        color: Colors.black.withValues(alpha: 0.55),
                        borderRadius: BorderRadius.circular(6),
                      ),
                      child: Text(
                        _telemetry,
                        style: const TextStyle(
                          color: Colors.greenAccent,
                          fontSize: 11,
                          fontFamily: 'monospace',
                        ),
                      ),
                    ),
                  ),
              ],
            ),
          ),
          if (!_fx.isSupported)
            const Padding(
              padding: EdgeInsets.all(8),
              child: Text(
                'Native effects pipeline not available — running passthrough.',
                style: TextStyle(color: Colors.orange),
                textAlign: TextAlign.center,
              ),
            ),
        ],
      ),
    );
  }
}