avatar_kit 1.0.0-beta.21
avatar_kit: ^1.0.0-beta.21 copied to clipboard
A Flutter plugin for AvatarKit, which provides high-performance avatar rendering with real-time and audio-driven capabilities.
example/lib/main.dart
import 'dart:async';
import 'dart:convert';
import 'package:flutter/material.dart' hide ConnectionState, Transform;
import 'package:flutter/services.dart';
import 'package:http/http.dart' as http;
import 'package:shared_preferences/shared_preferences.dart';
import 'package:avatar_kit/avatar_kit.dart';
const List<Map<String, String>> presetAvatars = [
{'name': 'Avatar 1', 'id': 'c067bb81-93cc-4a39-9622-9fb1c593cda6'},
{'name': 'Avatar 2', 'id': '4377b021-51b7-431f-87e1-60905b3138bc'},
{'name': 'Avatar 3', 'id': '9da199e2-520d-4d1b-96b0-a89ab23b9fb0'},
{'name': 'Avatar 4', 'id': 'c4908ab8-d5ce-4b29-9fac-ac80531af158'},
];
const List<String> audioFiles = ['audio_01.pcm', 'audio_02.pcm', 'audio_03.pcm'];
// --- AppConfig ---
class AppConfig {
String appID;
String sessionToken;
String userID;
String avatarID;
String environment;
int sampleRate;
String drivingServiceMode;
AppConfig({
this.appID = 'Demo',
this.sessionToken = '',
this.userID = '',
this.avatarID = 'c067bb81-93cc-4a39-9622-9fb1c593cda6',
this.environment = 'intl',
this.sampleRate = 16000,
this.drivingServiceMode = 'sdk',
});
static Future<AppConfig> load() async {
final prefs = await SharedPreferences.getInstance();
return AppConfig(
appID: prefs.getString('appID') ?? 'Demo',
sessionToken: prefs.getString('sessionToken') ?? '',
userID: prefs.getString('userID') ?? '',
avatarID: prefs.getString('avatarID') ?? presetAvatars[0]['id']!,
environment: prefs.getString('environment') ?? 'intl',
sampleRate: prefs.getInt('sampleRate') ?? 16000,
drivingServiceMode: prefs.getString('drivingServiceMode') ?? 'sdk',
);
}
Future<void> save() async {
final prefs = await SharedPreferences.getInstance();
await prefs.setString('appID', appID);
await prefs.setString('sessionToken', sessionToken);
await prefs.setString('userID', userID);
await prefs.setString('avatarID', avatarID);
await prefs.setString('environment', environment);
await prefs.setInt('sampleRate', sampleRate);
await prefs.setString('drivingServiceMode', drivingServiceMode);
}
String get avatarDisplayName {
for (final preset in presetAvatars) {
if (preset['id'] == avatarID) return preset['name']!;
}
return avatarID.length > 8 ? '${avatarID.substring(0, 8)}...' : avatarID;
}
}
// --- Main ---
void main() {
runApp(const AvatarKitApp());
}
class AvatarKitApp extends StatefulWidget {
const AvatarKitApp({super.key});
@override
State<AvatarKitApp> createState() => _AvatarKitAppState();
}
class _AvatarKitAppState extends State<AvatarKitApp> {
AppConfig? config;
@override
void initState() {
super.initState();
AppConfig.load().then((c) => setState(() => config = c));
}
@override
Widget build(BuildContext context) {
return MaterialApp(
theme: ThemeData(useMaterial3: true),
home: config == null
? const Scaffold(body: Center(child: CircularProgressIndicator()))
: DemoPage(config: config!),
);
}
}
// --- Demo Page (matches iOS ContentView) ---
class DemoPage extends StatefulWidget {
final AppConfig config;
const DemoPage({super.key, required this.config});
@override
State<DemoPage> createState() => _DemoPageState();
}
class _DemoPageState extends State<DemoPage> {
late AppConfig config;
// State
Avatar? _avatar;
AvatarController? _avatarController;
bool _isLoading = false;
String _loadingMessage = '';
double _progress = 0;
String? _errorMessage;
ConnectionState _connectionState = ConnectionState.disconnected;
ConversationState _conversationState = ConversationState.idle;
bool _isRenderingPaused = false;
bool _isSendingAudio = false;
String? _currentlyPlayingFile;
Timer? _sendAudioTimer;
bool get _isConnected => _connectionState == ConnectionState.connected;
@override
void initState() {
super.initState();
config = widget.config;
_initializeSDK();
}
@override
void dispose() {
_sendAudioTimer?.cancel();
super.dispose();
}
void _initializeSDK() {
final env = Environment.values.firstWhere(
(e) => e.name == config.environment,
orElse: () => Environment.intl,
);
final mode = config.drivingServiceMode == 'host'
? DrivingServiceMode.host
: DrivingServiceMode.sdk;
AvatarSDK.initialize(
appID: config.appID,
configuration: Configuration(
environment: env,
audioFormat: AudioFormat(sampleRate: config.sampleRate),
drivingServiceMode: mode,
logLevel: LogLevel.all,
),
);
if (config.sessionToken.isNotEmpty) {
AvatarSDK.setSessionToken(config.sessionToken);
}
if (config.userID.isNotEmpty) {
AvatarSDK.setUserID(config.userID);
}
}
// --- Settings ---
void _showSettings() async {
final result = await showModalBottomSheet<AppConfig>(
context: context,
isScrollControlled: true,
useSafeArea: true,
builder: (context) => ConfigSheet(config: config),
);
if (result != null) {
await result.save();
setState(() {
config = result;
_avatar = null;
_avatarController = null;
_connectionState = ConnectionState.disconnected;
_conversationState = ConversationState.idle;
_errorMessage = null;
_isRenderingPaused = false;
});
_initializeSDK();
}
}
// --- Avatar Loading ---
void _loadAvatar() async {
final avatarID = config.avatarID;
if (avatarID.isEmpty) {
setState(() => _errorMessage = 'Avatar ID is empty');
return;
}
setState(() {
_isLoading = true;
_errorMessage = null;
_loadingMessage = 'Loading...';
_progress = 0;
});
_initializeSDK();
try {
Avatar? avatar = await AvatarManager.shared.retrieve(id: avatarID);
if (avatar != null) {
setState(() => _loadingMessage = 'Loaded from cache');
} else {
avatar = await AvatarManager.shared.load(
id: avatarID,
onProgress: (progress) {
setState(() {
_progress = progress;
_loadingMessage = 'Downloading... ${(progress * 100).toInt()}%';
});
},
);
setState(() => _loadingMessage = 'Loaded');
}
setState(() {
_avatar = avatar;
_isLoading = false;
});
} on AvatarError catch (e) {
setState(() {
_isLoading = false;
_errorMessage = e.name;
_loadingMessage = 'Failed';
});
} catch (e) {
setState(() {
_isLoading = false;
_errorMessage = e.toString();
_loadingMessage = 'Failed';
});
}
}
// --- Platform View Created ---
void _onPlatformViewCreated(AvatarController controller) async {
controller.onFirstRendering = () {
debugPrint('onFirstRendering');
setState(() => _loadingMessage = 'Rendering');
};
controller.onConnectionState = (state, errorMessage) {
debugPrint('onConnectionState: ${state.name}');
setState(() {
_connectionState = state;
if (state == ConnectionState.failed && errorMessage != null) {
_errorMessage = errorMessage;
}
});
};
controller.onConversationState = (state) {
debugPrint('onConversationState: ${state.name}');
setState(() => _conversationState = state);
};
controller.onError = (error) {
debugPrint('onError: ${error.name}');
setState(() => _errorMessage = error.name);
};
await controller.setVolume(1.0);
await controller.setOpaque(false);
await controller.setContentTransform(Transform.identity);
final pointCount = await controller.pointCount();
debugPrint('pointCount: $pointCount');
setState(() => _avatarController = controller);
}
// --- Controls ---
void _start() async {
final token = await AvatarSDK.sessionToken();
final appID = await AvatarSDK.appID();
debugPrint('[Demo] start: appID=$appID, token=${token.isEmpty ? "EMPTY" : "${token.length} chars"}');
_avatarController?.start();
}
void _stop() {
_cancelAudioSending();
_avatarController?.close();
setState(() {
_connectionState = ConnectionState.disconnected;
_conversationState = ConversationState.idle;
});
}
void _interrupt() {
_cancelAudioSending();
_avatarController?.interrupt();
}
void _pauseRendering() {
_avatarController?.pauseRendering();
setState(() => _isRenderingPaused = true);
}
void _resumeRendering() {
_avatarController?.resumeRendering();
setState(() => _isRenderingPaused = false);
}
// --- Audio Sending ---
void _sendAudioFile(String filename) async {
if (!_isConnected || _avatarController == null) return;
_cancelAudioSending();
_avatarController!.interrupt();
final name = filename.replaceAll('.pcm', '');
try {
final data = await rootBundle.load('assets/$filename');
final audioData = data.buffer.asUint8List();
final chunkSize = config.sampleRate * 2; // 1 second of 16-bit mono PCM
setState(() {
_isSendingAudio = true;
_currentlyPlayingFile = filename;
});
int offset = 0;
_sendAudioTimer = Timer.periodic(const Duration(milliseconds: 100), (timer) {
if (offset >= audioData.length) {
timer.cancel();
setState(() {
_isSendingAudio = false;
_currentlyPlayingFile = null;
});
return;
}
final end = (offset + chunkSize).clamp(0, audioData.length);
final isLast = end >= audioData.length;
final chunk = audioData.sublist(offset, end);
_avatarController?.send(Uint8List.fromList(chunk), end: isLast);
offset = end;
});
} catch (e) {
setState(() {
_errorMessage = 'Cannot read $filename';
_isSendingAudio = false;
_currentlyPlayingFile = null;
});
}
}
void _cancelAudioSending() {
_sendAudioTimer?.cancel();
_sendAudioTimer = null;
setState(() {
_isSendingAudio = false;
_currentlyPlayingFile = null;
});
}
// --- Host Mode: Mock Data ---
final Map<int, _MockData> _mockData = {};
Future<void> _loadMockData() async {
if (_mockData.containsKey(config.sampleRate)) return;
try {
final response = await http.post(
Uri.parse('https://python-sdk-mock-demo.spatialwalk.cn/generate'),
headers: {'Content-Type': 'application/json'},
body: jsonEncode({'sample_rate': '${config.sampleRate}'}),
);
if (response.statusCode == 200) {
setState(() {
_mockData[config.sampleRate] = _MockData.fromJson(jsonDecode(response.body));
});
}
} catch (e) {
debugPrint('Fetching mock data failed: $e');
}
}
void _playMockStream() async {
final data = _mockData[config.sampleRate];
if (data == null || _avatarController == null) return;
final conversationID = await _avatarController!.yieldAudioData(
data.audio, end: true,
audioFormat: AudioFormat(sampleRate: config.sampleRate),
);
for (var i = 0; i < data.animations.length; i++) {
final animation = data.animations[i];
Future.delayed(Duration(seconds: i + 1), () {
_avatarController?.yieldAnimations([animation], conversationID: conversationID);
});
}
}
void _playMockAll() async {
final data = _mockData[config.sampleRate];
if (data == null || _avatarController == null) return;
final conversationID = await _avatarController!.yieldAudioData(
data.audio, end: true,
audioFormat: AudioFormat(sampleRate: config.sampleRate),
);
_avatarController!.yieldAnimations(data.animations, conversationID: conversationID);
}
// --- UI ---
@override
Widget build(BuildContext context) {
final isHostMode = config.drivingServiceMode == 'host';
return Scaffold(
appBar: AppBar(
title: Text('AvatarKit Demo${isHostMode ? ' (Host)' : ''}'),
actions: [
IconButton(icon: const Icon(Icons.settings), onPressed: _showSettings),
],
),
body: SafeArea(
child: Column(
children: [
// Avatar section
Expanded(flex: 3, child: _buildAvatarSection()),
if (_errorMessage != null)
Padding(
padding: const EdgeInsets.symmetric(horizontal: 16),
child: Text(_errorMessage!, style: const TextStyle(color: Colors.red, fontSize: 12)),
),
const Divider(height: 1),
// Status section
_buildStatusSection(),
const Divider(height: 1),
// Controls section
_buildControlsSection(),
const Divider(height: 1),
// Audio / Host section
Expanded(
flex: 2,
child: isHostMode ? _buildHostSection() : _buildAudioFilesSection(),
),
],
),
),
);
}
Widget _buildAvatarSection() {
if (_avatar != null) {
return AvatarWidget(
avatar: _avatar!,
onPlatformViewCreated: _onPlatformViewCreated,
);
}
if (_isLoading) {
return Center(
child: Column(
mainAxisSize: MainAxisSize.min,
children: [
const CircularProgressIndicator(),
const SizedBox(height: 16),
Text(_loadingMessage, style: const TextStyle(color: Colors.grey)),
if (_progress > 0) ...[
const SizedBox(height: 8),
SizedBox(width: 200, child: LinearProgressIndicator(value: _progress)),
],
],
),
);
}
return Center(
child: Column(
mainAxisSize: MainAxisSize.min,
children: [
Icon(Icons.person_outline, size: 60, color: Colors.grey[400]),
const SizedBox(height: 16),
Text('Tap Initialize to load avatar', style: TextStyle(color: Colors.grey[500])),
],
),
);
}
Widget _buildStatusSection() {
return Padding(
padding: const EdgeInsets.symmetric(horizontal: 16, vertical: 8),
child: Row(
children: [
_statusDot(_connectionStateColor),
const SizedBox(width: 4),
Text(_connectionStateName, style: const TextStyle(fontSize: 12)),
const SizedBox(width: 20),
_statusDot(_conversationStateColor),
const SizedBox(width: 4),
Text(_conversationState.name, style: const TextStyle(fontSize: 12)),
const Spacer(),
Text(config.avatarDisplayName,
style: const TextStyle(fontSize: 12, color: Colors.grey)),
],
),
);
}
Widget _statusDot(Color color) {
return Container(
width: 8, height: 8,
decoration: BoxDecoration(color: color, shape: BoxShape.circle),
);
}
String get _connectionStateName {
switch (_connectionState) {
case ConnectionState.disconnected: return 'Disconnected';
case ConnectionState.connecting: return 'Connecting';
case ConnectionState.connected: return 'Connected';
case ConnectionState.failed: return 'Failed';
}
}
Color get _connectionStateColor {
switch (_connectionState) {
case ConnectionState.disconnected: return Colors.grey;
case ConnectionState.connecting: return Colors.orange;
case ConnectionState.connected: return Colors.green;
case ConnectionState.failed: return Colors.red;
}
}
Color get _conversationStateColor {
switch (_conversationState) {
case ConversationState.idle: return Colors.grey;
case ConversationState.playing: return Colors.green;
case ConversationState.paused: return Colors.orange;
}
}
Widget _buildControlsSection() {
return Padding(
padding: const EdgeInsets.symmetric(horizontal: 16, vertical: 8),
child: Column(
children: [
Row(
children: [
_ctrlBtn('Initialize', Icons.download, _loadAvatar, enabled: !_isLoading),
const SizedBox(width: 8),
_isConnected
? _ctrlBtn('Stop', Icons.stop_circle_outlined, _stop)
: _ctrlBtn('Start', Icons.play_circle_outlined, _start, enabled: _avatar != null),
],
),
const SizedBox(height: 8),
Row(
children: [
_isRenderingPaused
? _ctrlBtn('Show', Icons.visibility, _resumeRendering)
: _ctrlBtn('Hide', Icons.visibility_off, _pauseRendering, enabled: _avatar != null),
const SizedBox(width: 8),
_ctrlBtn('Interrupt', Icons.cancel_outlined, _interrupt, enabled: _isConnected),
],
),
],
),
);
}
Widget _ctrlBtn(String label, IconData icon, VoidCallback onPressed, {bool enabled = true}) {
return Expanded(
child: OutlinedButton(
onPressed: enabled ? onPressed : null,
child: Column(
mainAxisSize: MainAxisSize.min,
children: [
Icon(icon, size: 20),
const SizedBox(height: 2),
Text(label, style: const TextStyle(fontSize: 11)),
],
),
),
);
}
// --- Audio Files Section (SDK mode) ---
Widget _buildAudioFilesSection() {
return Column(
crossAxisAlignment: CrossAxisAlignment.start,
children: [
Padding(
padding: const EdgeInsets.symmetric(horizontal: 16, vertical: 6),
child: Row(
children: [
const Text('Audio Files', style: TextStyle(fontSize: 12, color: Colors.grey)),
const Spacer(),
if (_isSendingAudio) const SizedBox(width: 16, height: 16, child: CircularProgressIndicator(strokeWidth: 2)),
],
),
),
Expanded(
child: ListView.builder(
itemCount: audioFiles.length,
itemBuilder: (context, index) {
final file = audioFiles[index];
final isPlaying = _currentlyPlayingFile == file;
return ListTile(
leading: const Icon(Icons.graphic_eq, size: 20),
title: Text(file, style: const TextStyle(fontSize: 14)),
trailing: isPlaying ? const Icon(Icons.volume_up, color: Colors.blue, size: 20) : null,
dense: true,
enabled: _isConnected && !_isSendingAudio,
onTap: () => _sendAudioFile(file),
);
},
),
),
],
);
}
// --- Host Section ---
Widget _buildHostSection() {
final hasData = _mockData.containsKey(config.sampleRate);
return Padding(
padding: const EdgeInsets.symmetric(horizontal: 16, vertical: 8),
child: Column(
children: [
Text('Sample Rate: ${config.sampleRate} Hz', style: const TextStyle(fontSize: 12, color: Colors.grey)),
const SizedBox(height: 8),
Row(
children: [
Expanded(
child: OutlinedButton(
onPressed: _loadMockData,
child: Text(hasData ? 'Mock Data Loaded' : 'Load Mock Data', style: const TextStyle(fontSize: 12)),
),
),
],
),
const SizedBox(height: 8),
Row(
children: [
Expanded(
child: OutlinedButton(
onPressed: hasData ? _playMockStream : null,
child: const Text('Play Stream', style: TextStyle(fontSize: 12)),
),
),
const SizedBox(width: 8),
Expanded(
child: OutlinedButton(
onPressed: hasData ? _playMockAll : null,
child: const Text('Playback', style: TextStyle(fontSize: 12)),
),
),
],
),
const SizedBox(height: 8),
Row(
children: [
Expanded(
child: OutlinedButton(
onPressed: _isConnected ? _interrupt : null,
child: const Text('Interrupt', style: TextStyle(fontSize: 12)),
),
),
],
),
const Spacer(),
Text('Conversation: ${_conversationState.name}', style: const TextStyle(fontSize: 12, color: Colors.grey)),
],
),
);
}
}
// --- Config Sheet ---
class ConfigSheet extends StatefulWidget {
final AppConfig config;
const ConfigSheet({super.key, required this.config});
@override
State<ConfigSheet> createState() => _ConfigSheetState();
}
class _ConfigSheetState extends State<ConfigSheet> {
late TextEditingController _appIDController;
late TextEditingController _tokenController;
late TextEditingController _userIDController;
late TextEditingController _customAvatarIDController;
late String _selectedEnvironment;
late int _selectedSampleRate;
late String _selectedAvatarID;
late String _selectedMode;
bool _isCustomAvatar = false;
@override
void initState() {
super.initState();
_appIDController = TextEditingController(text: widget.config.appID);
_tokenController = TextEditingController(text: widget.config.sessionToken);
_userIDController = TextEditingController(text: widget.config.userID);
_selectedEnvironment = widget.config.environment;
_selectedSampleRate = widget.config.sampleRate;
_selectedAvatarID = widget.config.avatarID;
_selectedMode = widget.config.drivingServiceMode;
_isCustomAvatar = !presetAvatars.any((a) => a['id'] == _selectedAvatarID);
_customAvatarIDController = TextEditingController(
text: _isCustomAvatar ? _selectedAvatarID : '',
);
}
@override
void dispose() {
_appIDController.dispose();
_tokenController.dispose();
_userIDController.dispose();
_customAvatarIDController.dispose();
super.dispose();
}
void _save() {
final avatarID = _isCustomAvatar
? _customAvatarIDController.text.trim()
: _selectedAvatarID;
if (avatarID.isEmpty) return;
Navigator.pop(context, AppConfig(
appID: _appIDController.text.trim(),
sessionToken: _tokenController.text.trim(),
userID: _userIDController.text.trim(),
avatarID: avatarID,
environment: _selectedEnvironment,
sampleRate: _selectedSampleRate,
drivingServiceMode: _selectedMode,
));
}
@override
Widget build(BuildContext context) {
return Scaffold(
appBar: AppBar(
title: const Text('Settings'),
leading: TextButton(
onPressed: () => Navigator.pop(context),
child: const Text('Cancel'),
),
leadingWidth: 80,
actions: [
TextButton(onPressed: _save, child: const Text('Save')),
],
),
body: ListView(
children: [
// SDK Configuration
_sectionHeader('SDK Configuration'),
_textField(_appIDController, 'App ID'),
_textField(_tokenController, 'Session Token', obscure: true),
_textField(_userIDController, 'User ID'),
_dropdownTile('Environment', _selectedEnvironment, {
'intl': 'International',
'cn': 'China',
}, (v) => setState(() => _selectedEnvironment = v)),
_dropdownTile('Sample Rate', _selectedSampleRate.toString(),
{ for (var r in [8000, 16000, 22050, 24000, 32000, 44100, 48000]) r.toString(): '$r Hz' },
(v) => setState(() => _selectedSampleRate = int.parse(v)),
),
_dropdownTile('Driving Service Mode', _selectedMode, {
'sdk': 'SDK',
'host': 'Host',
}, (v) => setState(() => _selectedMode = v)),
// Avatar
_sectionHeader('Avatar'),
...presetAvatars.map((avatar) => RadioListTile<String>(
title: Text(avatar['name']!),
subtitle: Text(avatar['id']!, style: const TextStyle(fontSize: 10)),
value: avatar['id']!,
groupValue: _isCustomAvatar ? null : _selectedAvatarID,
onChanged: (value) => setState(() {
_isCustomAvatar = false;
_selectedAvatarID = value!;
}),
dense: true,
)),
RadioListTile<bool>(
title: const Text('Custom'),
value: true,
groupValue: _isCustomAvatar ? true : null,
onChanged: (_) => setState(() => _isCustomAvatar = true),
dense: true,
),
if (_isCustomAvatar)
Padding(
padding: const EdgeInsets.symmetric(horizontal: 16),
child: TextField(
controller: _customAvatarIDController,
decoration: const InputDecoration(labelText: 'Custom Avatar ID'),
),
),
const SizedBox(height: 32),
],
),
);
}
Widget _sectionHeader(String title) {
return Padding(
padding: const EdgeInsets.fromLTRB(16, 16, 16, 4),
child: Text(title, style: TextStyle(fontSize: 13, fontWeight: FontWeight.w600, color: Colors.grey[600])),
);
}
Widget _textField(TextEditingController controller, String label, {bool obscure = false}) {
return Padding(
padding: const EdgeInsets.symmetric(horizontal: 16, vertical: 4),
child: TextField(
controller: controller,
decoration: InputDecoration(labelText: label, isDense: true),
obscureText: obscure,
autocorrect: false,
),
);
}
Widget _dropdownTile(String label, String value, Map<String, String> options, ValueChanged<String> onChanged) {
return ListTile(
title: Text(label, style: const TextStyle(fontSize: 14)),
trailing: DropdownButton<String>(
value: value,
underline: const SizedBox(),
items: options.entries.map((e) => DropdownMenuItem(value: e.key, child: Text(e.value))).toList(),
onChanged: (v) { if (v != null) onChanged(v); },
),
dense: true,
);
}
}
// --- Mock Data ---
class _MockData {
final Uint8List audio;
final List<Uint8List> animations;
_MockData({required this.audio, required this.animations});
factory _MockData.fromJson(Map<String, dynamic> json) {
return _MockData(
audio: base64Decode(json['audio_base64']),
animations: (json['animation_messages_base64'] as List<dynamic>)
.map<Uint8List>((animation) => base64Decode(animation as String))
.toList(),
);
}
}