Guest User

Untitled

a guest
Dec 15th, 2024
458
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 39.95 KB | None | 0 0
  1. // live.dart
  2. import 'dart:convert';
  3. import 'dart:typed_data';
  4.  
  5. import 'package:collection/collection.dart';
  6. import 'package:fonnx/models/pyannote/pyannote.dart';
  7. import 'package:freezed_annotation/freezed_annotation.dart';
  8. import 'package:hooks_riverpod/hooks_riverpod.dart';
  9. import 'package:loggy/loggy.dart' hide LogLevel;
  10. import 'package:telosnex/agent/agent_script_select.dart';
  11. import 'package:telosnex/chat/models/tool.dart' hide Tool;
  12. import 'package:telosnex/chat/realtime/audio_ring_buffer.dart';
  13. import 'package:telosnex/chat/realtime/live_history_string.dart';
  14. import 'package:telosnex/chat/realtime/map_has_crosstalk.dart';
  15. import 'package:telosnex/chat/realtime/realtime_models.dart' hide Conversation;
  16. import 'package:telosnex/misc/pcm_to_wav.dart';
  17. import 'package:telosnex/models/companion_providers.dart';
  18. import 'package:telosnex/models/conversation.dart';
  19. import 'package:telosnex/chat/models/role.dart' as tlsnx_role;
  20. import 'package:telosnex/models/script.dart';
  21. import 'package:telosnex/persistence/document_manager.dart';
  22. import 'package:telosnex/persistence/path_providers.dart';
  23. import 'package:telosnex/run/run.dart';
  24. import 'package:telosnex/run/run_status.dart';
  25. import 'package:telosnex/stt/pyannote_path.dart';
  26. import 'package:telosnex/sync/rate_limited_publisher.dart';
  27. import 'package:telosnex/widgets/patrol_keys.dart';
  28.  
  29. import 'live_architecture.dart';
  30. import 'live_event_handlers.dart';
  31. import 'live_internal_handlers.dart';
  32. import 'live_internal_events.dart';
  33. import 'package:riverpod_annotation/riverpod_annotation.dart';
  34. import 'package:telosnex/loggy/live_loggy.dart';
  35. import 'package:telosnex/chat/realtime/realtime_socket.dart';
  36. import 'package:telosnex/models/conversation_manager.dart';
  37. import 'package:telosnex/models/api_keys.dart';
  38. import 'package:telosnex/run/run_manager.dart';
  39. import 'package:telosnex/models/script_manager.dart';
  40. import 'package:telosnex/models/settings.dart';
  41. import 'package:telosnex/stt/microphone_service.dart';
  42. import 'package:flutter_pcm_sound/flutter_pcm_sound.dart';
  43. import 'dart:async';
  44. import 'package:telosnex/models/loom_step.dart'; // For LiveState initial steps
  45.  
  46. part 'live.freezed.dart';
  47. part 'live.g.dart';
  48.  
  49. enum LiveEvent { started, stopped, error }
  50.  
  51. @freezed
  52. class LiveState with _$LiveState {
  53. const factory LiveState({
  54. required bool isRunning,
  55. required bool isRemoteTalking,
  56. required bool isSocketConnected,
  57. required bool didRemoteTalkInFirstSecond,
  58. required Map<String, LoomStep> itemIdToLoomStep,
  59. // If *any* audio in the first second is detected, it is assumed that it is
  60. // echo from the assistant. If thats true, that implies headphones could not
  61. // be connected. With this information, the crosstalk detection algorithm
  62. // can make better decisions.
  63. //
  64. // Motivated by the assistant reading a poem
  65. // dramatically, which at second 21, rolled such that there were no segments
  66. // in the first 1.02 seconds. This was evaluated as a headphone interruption.
  67. required bool isMicMutedByUser,
  68. required bool isSpeakerMutedByUser,
  69. String? conversationId,
  70. @Default([]) List<LiveEvent> events,
  71. }) = _LiveState;
  72.  
  73. factory LiveState.initial() => LiveState(
  74. isRunning: false,
  75. isRemoteTalking: false,
  76. isSocketConnected: false,
  77. didRemoteTalkInFirstSecond: false,
  78. itemIdToLoomStep: {},
  79. isMicMutedByUser: false,
  80. isSpeakerMutedByUser: false,
  81. );
  82. }
  83.  
  84. const _kLiveDebug = true;
  85.  
  86. void liveLog(String Function() message) {
  87. if (_kLiveDebug) {
  88. logDebug('[LIVE] ${message()}');
  89. }
  90. }
  91.  
  92. void liveLogError(String Function() message) {
  93. logError('[LIVE] ${message()}');
  94. }
  95.  
  96. @Riverpod(
  97. dependencies: [ScriptManager, apiKeys, ConversationManager, RunManager])
  98. class LiveNotifier extends _$LiveNotifier with LiveLoggy {
  99. static const kInjectedStepIdPrefix = 'ClientC_';
  100. // static const kFpcmLogLevel = LogLevel.none;
  101.  
  102. RealtimeSocket? _socket;
  103. StreamSubscription? _socketEventsSubscription;
  104. StreamSubscription? _microphoneSubscription;
  105. MicrophoneService? _microphone;
  106. AudioRingBuffer? _micDataWhenRemoteTalkingBuffer;
  107. AudioRingBuffer? _micDataWhenSocketConnectingBuffer;
  108.  
  109. final Set<String> _skippedAssistantItemIds = {};
  110. Pyannote? _pyannote; // Pyannote model
  111. ThrottledPublisher? _pyannoteThrottler;
  112. BytesBuilder _lastAudioPlayed = BytesBuilder();
  113. BytesBuilder _lastAudioFromAssistant = BytesBuilder();
  114.  
  115. @override
  116. FutureOr<LiveState> build() {
  117. ref.onDispose(() => stop());
  118. _lastAudioPlayed = BytesBuilder();
  119. _lastAudioFromAssistant = BytesBuilder();
  120. ref.listen(companionScriptIdProvider, (previous, next) {
  121. if (!state.value!.isRunning) return;
  122. final commands = [RefreshSessionInstructionsCommand(next)];
  123. _processCommands(commands);
  124. });
  125. return LiveState.initial();
  126. }
  127.  
  128. void muteMic() {
  129. final event = MicMuteEvent(isMuted: true);
  130. final result = handleInternalEvent(state.value!, event);
  131. state = AsyncData(result.newState);
  132. _processCommands(result.commands);
  133. }
  134.  
  135. void unmuteMic() {
  136. final event = MicMuteEvent(isMuted: false);
  137. final result = handleInternalEvent(state.value!, event);
  138. state = AsyncData(result.newState);
  139. _processCommands(result.commands);
  140. }
  141.  
  142. void muteSpeaker() {
  143. final event = SpeakerMuteEvent(isMuted: true);
  144. final result = handleInternalEvent(state.value!, event);
  145. state = AsyncData(result.newState);
  146. _processCommands(result.commands);
  147. }
  148.  
  149. void unmuteSpeaker() {
  150. final event = SpeakerMuteEvent(isMuted: false);
  151. final result = handleInternalEvent(state.value!, event);
  152. state = AsyncData(result.newState);
  153. _processCommands(result.commands);
  154. }
  155.  
  156. void skipAssistant() {
  157. final lastAssistantItemId = state.value!.itemIdToLoomStep.entries
  158. .lastWhereOrNull((e) =>
  159. e.value is LoomChatCompletionText &&
  160. (e.value as LoomChatCompletionText).role ==
  161. tlsnx_role.Role.assistant)
  162. ?.key;
  163.  
  164. if (lastAssistantItemId == null) {
  165. liveLogError(() => 'No assistant item found to skip');
  166. return;
  167. }
  168. _skippedAssistantItemIds.add(lastAssistantItemId);
  169. final event = AssistantSpeechStopRequestedEvent(lastAssistantItemId, 10000);
  170. final result = handleInternalEvent(state.value!, event);
  171. state = AsyncData(result.newState);
  172. _processCommands(result.commands);
  173. }
  174.  
  175. Future<String> start({
  176. required MicrophoneService microphone,
  177. required String initialPrompt,
  178. required String? chatId,
  179. }) async {
  180. _lastAudioPlayed = BytesBuilder();
  181. _lastAudioFromAssistant = BytesBuilder();
  182. await _microphone?.stop();
  183. _micDataWhenRemoteTalkingBuffer = AudioRingBuffer(
  184. (MicrophoneConfig.kBitsPerSample / 8).ceil() * /* bytes per sample */
  185. MicrophoneConfig.kSampleRate /* samples per second */ *
  186. 8 /* seconds */);
  187. _micDataWhenSocketConnectingBuffer = AudioRingBuffer(
  188. (MicrophoneConfig.kBitsPerSample / 8).ceil() * /* bytes per sample */
  189. MicrophoneConfig.kSampleRate /* samples per second */ *
  190. 8 /* seconds */);
  191. _microphone = microphone;
  192. _pyannote ??= Pyannote.load(await getPyannoteModelPath());
  193. _pyannoteThrottler =
  194. ThrottledPublisher(interval: const Duration(milliseconds: 300));
  195.  
  196. // Trigger SessionStartRequestedEvent
  197. final event = SessionStartRequestedEvent(
  198. initialPrompt,
  199. chatId,
  200. ref.read(apiKeysProvider),
  201. );
  202. final result = handleInternalEvent(state.value!, event);
  203. state = AsyncData(result.newState);
  204. _processCommands(result.commands);
  205. return state.valueOrNull!.conversationId!;
  206. }
  207.  
  208. void stop() {
  209. final event = LiveStopCalledEvent();
  210. final result = handleInternalEvent(state.value!, event);
  211. state = AsyncData(result.newState);
  212. _processCommands(result.commands);
  213. }
  214.  
  215. void _onSocketEvent(ServerEvent event) {
  216. final result = handleServerEvent(state.value!, event);
  217. state = AsyncData(result.newState);
  218. _processCommands(result.commands);
  219. }
  220.  
  221. void _processCommands(List<DomainCommand> commands) async {
  222. if (commands.isEmpty) return;
  223. if (commands.length == 1 &&
  224. (commands.first is SendSocketMessageCommand ||
  225. // ignore: no-empty-block
  226. commands.first is PersistConversationCommand)) {
  227. // avoiding log spam from mic uploads temporarily while debugging
  228. } else {
  229. liveLog(() =>
  230. '_processCommands: ${commands.map((e) => e.runtimeType).join(', ')}');
  231. }
  232. for (final cmd in commands) {
  233. switch (cmd) {
  234. case CacheLastAssistantAudioCommand _:
  235. _lastAudioFromAssistant = _lastAudioPlayed;
  236. _lastAudioPlayed = BytesBuilder();
  237. case RefreshSessionInstructionsCommand _:
  238. _sendInitialPrompt();
  239. case ClearRemoteSpeakingMicBufferCommand _:
  240. _micDataWhenRemoteTalkingBuffer?.clear();
  241. case PlayAudioCommand c:
  242. if (_skippedAssistantItemIds.contains(c.itemId)) {
  243. liveLog(() => 'Skipping audio for item ${c.itemId}');
  244. return;
  245. }
  246. if (state.valueOrNull?.isSpeakerMutedByUser == true) {
  247. liveLog(() =>
  248. 'Speaker muted by user, not playing audio for item ${c.itemId}');
  249. return;
  250. }
  251. _lastAudioPlayed.add(c.bytes);
  252. final Int16List samples = c.bytes.buffer.asInt16List();
  253. // `await` is crucial to avoid a ton of wasted time in debug mode on
  254. // overflow logs that aren't shown. https://api.flutter.dev/flutter/dart-ui/ChannelBuffers-class.html
  255. // It is likely these logs may also be a real issue since they concern the
  256. // platform channel being backed up with messages and having to overwrite
  257. // ones that have not been processed.
  258. await FlutterPcmSound.feed(
  259. PcmArrayInt16(bytes: samples.buffer.asByteData()));
  260. case SetConversationCommand c:
  261. final maybeChatId = c.id;
  262. final convoMgr = ref.read(conversationManagerProvider.notifier);
  263. final existing =
  264. maybeChatId == null ? null : convoMgr.getById(maybeChatId);
  265. final Conversation chat;
  266. if (existing != null) {
  267. chat = existing;
  268. } else {
  269. chat = Conversation(
  270. id: LoomStep.newId(),
  271. modifiedAt: DateTime.now(),
  272. syncLevel:
  273. ref.read(settingsNotifierProvider).syncLevelForNewObjects,
  274. );
  275. unawaited(convoMgr.upsert(chat));
  276. }
  277. state = AsyncData(state.value!.copyWith(conversationId: chat.id));
  278. case PersistConversationCommand c:
  279. final convoMgr = ref.read(conversationManagerProvider.notifier);
  280. final convo = convoMgr.getById(c.conversationId);
  281. if (convo == null) {
  282. liveLogError(() =>
  283. 'PersistConversationCommand: Conversation not found for id ${c.conversationId}');
  284. return;
  285. }
  286. final List<LoomStep> makeEmptyMessagesInvisible = c.steps
  287. .map((e) {
  288. if (e is LoomChatCompletionText &&
  289. e.isVisible &&
  290. e.text.isEmpty) {
  291. return e.copyWith(isVisible: false);
  292. }
  293. if (e is LoomChatCompletionText &&
  294. e.text.isNotEmpty &&
  295. !e.isVisible) {
  296. return e.copyWith(isVisible: false);
  297. }
  298. if (e is LoomChatDialogue && e.text.isEmpty) {
  299. return e.copyWith(isVisible: false);
  300. }
  301. if (e is LoomChatDialogue &&
  302. e.text.isNotEmpty &&
  303. !e.isVisible) {
  304. return e.copyWith(isVisible: true);
  305. }
  306. return e;
  307. })
  308. .toList()
  309. .cast<LoomStep>();
  310. final newConvo = convo.mergeSteps(makeEmptyMessagesInvisible);
  311. unawaited(convoMgr.upsert(newConvo));
  312. case SendSocketMessageCommand c:
  313. // liveLog(() => '${c.message.runtimeType} sending socket message');
  314. _socket?.sendEvent(c.message);
  315. case TruncateAssistantMessageCommand c:
  316. _socket?.sendEvent(RealtimeEvent.conversationItemTruncate(
  317. itemId: c.itemId,
  318. contentIndex: c.contentIndex,
  319. audioEndMs: c.audioEndMs));
  320. case RunScriptCommand c:
  321. _executeRunScript(c.input, c.serverEvent);
  322. case ResetAudioSystemCommand _:
  323. _resetAiAudioPlayback();
  324. case ConnectSocketCommand c:
  325. await _connectSocket(c.apiKeys, c.isReconnect);
  326. case DisconnectSocketCommand _:
  327. await _disconnectSocket();
  328. case StartMicrophoneCommand c:
  329. _startMicrophone(c.config);
  330. case StopMicrophoneCommand _:
  331. await _stopMicrophone();
  332. case LoadPyannoteModelCommand _:
  333. await _loadPyannoteModel();
  334. case BufferMicDataWhenRemoteTalkingCommand c:
  335. _bufferMicDataWhenRemoteTalking(c.bytes);
  336. case BufferMicDataWhenSocketDisconnectedCommand c:
  337. _bufferMicDataWhenSocketDisconnected(c.bytes);
  338. }
  339. }
  340. }
  341.  
  342. void _bufferMicDataWhenSocketDisconnected(Uint8List bytes) {
  343. liveLog(() => 'Buffering mic data while socket disconnected.');
  344. _micDataWhenSocketConnectingBuffer?.write(bytes);
  345. }
  346.  
  347. void _bufferMicDataWhenRemoteTalking(Uint8List bytes) {
  348. _micDataWhenRemoteTalkingBuffer?.write(bytes);
  349.  
  350. // ignore: avoid-passing-async-when-sync-expected
  351. _pyannoteThrottler!.publish(() async {
  352. if (state.value?.isRemoteTalking == false) {
  353. // liveLog(() =>
  354. // '[_bufferMicDataWhenRemoteTalking] Stream publisher checking *after* assistant stopped talking.');
  355. _micDataWhenRemoteTalkingBuffer?.clear();
  356. return;
  357. }
  358. final bufferedMicData =
  359. _micDataWhenRemoteTalkingBuffer!.getCurrentSamples();
  360. if (bufferedMicData.isEmpty) {
  361. // Needed because pyannote expects *some* audio, it does sublist
  362. // operations while calculating windows that ultimately leads to an
  363. // error of "Failed to process audio in isolate" from FONNX due to
  364. // a List.sublist() call in FONNX.
  365. liveLogError(
  366. () => '[_bufferMicDataWhenRemoteTalking] No data to process');
  367. return;
  368. }
  369. final floats = resampleTo16k(pcmSamplesToFloat32(bufferedMicData));
  370. final durationInMs = (bufferedMicData.length /
  371. (MicrophoneConfig.kBitsPerSample / 8) /
  372. MicrophoneConfig.kSampleRate *
  373. 1000)
  374. .round();
  375. final crosstalkResultAsMap = await _pyannote!.process(floats);
  376. final howLongRemoteTalkingMs = _micDataWhenRemoteTalkingBuffer!
  377. .bytesWrittenSinceLastClear
  378. .toDouble() ~/
  379. (MicrophoneConfig.kBitsPerSample / 8) ~/
  380. MicrophoneConfig.kSampleRate *
  381. 1000;
  382. final micDataSizeInMs = _micDataWhenRemoteTalkingBuffer!.capacityInBytes /
  383. (MicrophoneConfig.kBitsPerSample / 8) /
  384. MicrophoneConfig.kSampleRate *
  385. 1000;
  386. if (!state.value!.didRemoteTalkInFirstSecond &&
  387. howLongRemoteTalkingMs < micDataSizeInMs) {
  388. final hasRemoteTalkedAtLeast2Seconds = howLongRemoteTalkingMs > 2000;
  389. final anySegmentInFirstSecond =
  390. crosstalkResultAsMap.any((s) => s['start'] < 1000);
  391. if (hasRemoteTalkedAtLeast2Seconds && anySegmentInFirstSecond) {
  392. // liveLog(() =>
  393. // 'Remote talked for at least 2 seconds and has segments in the first second. From here on out, we assume there is echo, therefore the user cannot be using headphones.');
  394. state = AsyncData(
  395. state.value!.copyWith(didRemoteTalkInFirstSecond: true));
  396. }
  397. }
  398. final crosstalkResult = crosstalkResultAsMap.detectCrosstalk(
  399. areHeadphonesPossible: !state.value!.didRemoteTalkInFirstSecond,
  400. isDefinitelyHeadphones: state.value!.isSpeakerMutedByUser,
  401. );
  402. // liveLog(() =>
  403. // 'Crosstalk result: $crosstalkResult. Duration analyzed: $durationInMs ms. As map: $crosstalkResultAsMap');
  404. // ignore: dead_code
  405. if (false && durationInMs > 2000) {
  406. unawaited(writeWavFile(
  407. '${PathProvider.documentsDirectory}/${crosstalkResult.hasCrosstalk}--${DateTime.now().millisecondsSinceEpoch}.wav',
  408. floats,
  409. sampleRate: 16000));
  410. }
  411. if (crosstalkResult.hasCrosstalk) {
  412. if (durationInMs < 2000) {
  413. // We don't have confidence the models results will be stable &
  414. // accurate until it has at least 2 seconds of audio.
  415. liveLog(() => 'Crosstalk detected but duration too short');
  416. return;
  417. }
  418.  
  419. final audioEndMs = (crosstalkResult.crosstalkStartTime! *
  420. MicrophoneConfig.kSampleRate /
  421. MicrophoneConfig.kSampleRateOpenAiRealtime)
  422. .round();
  423.  
  424. final result = handleInternalEvent(
  425. state.valueOrNull!,
  426. CrosstalkDetectedEvent(
  427. audioEndMs: audioEndMs,
  428. bufferedMicData: bufferedMicData.buffer.asUint8List(),
  429. ),
  430. );
  431. state = AsyncData(result.newState);
  432. _processCommands(result.commands);
  433. }
  434. });
  435. }
  436.  
  437. void _executeRunScript(LoomChatCompletionFunction input,
  438. ResponseFunctionCallArgumentsDoneEvent serverEvent) async {
  439. final isScriptSelect =
  440. input.functionName == AgentScriptSelect.selectScriptToolName;
  441. final isScriptRun = input.functionName == kUseScriptToolname;
  442. final Script script;
  443. if (isScriptRun) {
  444. final scriptId = ref.read(companionScriptIdProvider);
  445. final maybeScript = ref
  446. .read(scriptManagerProvider)
  447. .valueOrNull
  448. ?.firstWhereOrNull((e) => e.id == scriptId);
  449. if (maybeScript == null) {
  450. liveLogError(() =>
  451. 'Script with id $scriptId not found. Script manager: ${ref.read(scriptManagerProvider).valueOrNull}');
  452. return;
  453. }
  454. script = maybeScript;
  455. } else if (isScriptSelect) {
  456. script = AgentScriptSelect.scriptForFunctionArguments(
  457. input, ref.read(scriptManagerProvider).valueOrNull ?? []);
  458. } else {
  459. liveLogError(
  460. () => 'Cannot determine script for RunScriptCommand: $input');
  461. return;
  462. }
  463.  
  464. final runManager = ref.read(runManagerProvider.notifier);
  465. final streamController = await runManager.run(
  466. imageMessages: [],
  467. companionFiles: ref.read(companionFilesObjectProvider),
  468. conversationId: state.value!.conversationId!,
  469. preprompt: [],
  470. prompt: runPromptStep(
  471. input.functionArguments[kScriptSelectPromptFunctionKey] ?? ''),
  472. script: script,
  473. history: ref.read(companionContextInputProvider),
  474. conversationManager: ref.read(conversationManagerProvider.notifier),
  475. documentManager: ref.read(documentManagerProvider.notifier),
  476. initialStatus: RunStatusPickingScript(script.label),
  477. // Getting the rewritten prompt from the tool output helps significantly,
  478. // the user prompt transcript often is not available yet when Run starts.
  479. //
  480. // However, the rewritten prompt is also added to the chat history. Lets
  481. // make it invisible so there's no duplication.
  482. isPromptVisible: false,
  483. );
  484. if (streamController == null) {
  485. loggy.error('Run manager did not return a stream controller');
  486. return;
  487. }
  488. final lastRunOutput = await streamController.stream.last;
  489. final visibleSteps = lastRunOutput.steps
  490. .whereType<LoomChatCompletionText>()
  491. .where((e) => e.isVisible && e.text.isNotEmpty)
  492. .toList();
  493. final concatenatedOutput =
  494. visibleSteps.map((e) => '${e.role.label}: ${e.text}').join('\n\n');
  495. final socket = _socket;
  496. if (socket == null) {
  497. loggy.error('Socket is null');
  498. return;
  499. }
  500.  
  501. final injectedResponseCreatedItemId =
  502. '$kInjectedStepIdPrefix${DateTime.now().millisecondsSinceEpoch}';
  503. socket.sendEvent(
  504. ConversationItemCreateEvent(
  505. item: Item.functionCallOutput(
  506. id: injectedResponseCreatedItemId,
  507. callId: serverEvent.callId,
  508. output: '# Output\n$concatenatedOutput\n\n\n',
  509. ),
  510. ),
  511. );
  512. // TODO: Adding this to test if it prevents cases where it apologizes and does the search again
  513. // after the script has been run and we add these steps.
  514. await socket.events.firstWhere((e) => e is ConversationItemCreatedEvent);
  515. final injectedRequestResponseEventId =
  516. '$kInjectedStepIdPrefix${DateTime.now().millisecondsSinceEpoch}';
  517. final eventToSend = ResponseCreateEvent(
  518. eventId: injectedRequestResponseEventId,
  519. response: Response(
  520. // id of response has to be null for ResponseCreateEvent, otherwise,
  521. // OpenAI server errors.
  522. id: null,
  523. // Can only be audio & text, or, text. Audio only is not allowed, which
  524. // is unfortunate, as it would allow us to avoid injecting another text
  525. // response that copies the existing script output.
  526. modalities: ['audio', 'text'],
  527. maxOutputTokens: 4096,
  528. temperature:
  529. createDefaultSession(initialPrompt: '', ref: ref).temperature,
  530. instructions:
  531. '# Task \nAbove is the output of the script. Please vocally respond to the user.',
  532. ),
  533. );
  534. socket.sendEvent(eventToSend);
  535. }
  536.  
  537. void _resetAiAudioPlayback() {
  538. if (P.isTesting) {
  539. // Avoid MissingPluginException(No implementation found for method setLogLevel on channel flutter_pcm_sound/methods)
  540. // affects ~30 of 1200 tests.
  541. return;
  542. }
  543. if (state.valueOrNull?.isRemoteTalking == true) {
  544. liveLog(() => '[LIVE] _resetAudioSystem isRemoteTalking = false');
  545. state = AsyncData(state.value!.copyWith(isRemoteTalking: false));
  546. }
  547. liveLog(() => 'Resetting audio system');
  548. FlutterPcmSound.release();
  549. FlutterPcmSound.setup(
  550. sampleRate: 24000,
  551. channelCount: 1,
  552. iosAudioCategory: IosAudioCategory.playAndRecord,
  553. );
  554. // This is delicate - the feed callback is how we decide when the assistant
  555. // stops talking. If we set a higher feed threshold, say, 8000 frames,
  556. // there's significant slippage between the actual speech stop and when we
  557. // expect it to stop. We don't have a back buffer to feed audio anyway,
  558. // we give it as soon as we get it, so might as well use feed threshold 0.
  559. FlutterPcmSound.setFeedThreshold(0);
  560. FlutterPcmSound.setLogLevel(LogLevel.none);
  561. FlutterPcmSound.setFeedCallback((index) {
  562. if (state.value!.isRemoteTalking) {
  563. liveLog(() => 'Remote talking will end in 450 ms.');
  564.  
  565. Future.delayed(const Duration(milliseconds: 450), () {
  566. state = AsyncData(state.value!.copyWith(isRemoteTalking: false));
  567. liveLog(() => 'Remote talking ended');
  568. });
  569. }
  570. });
  571. }
  572.  
  573. Future<void> _connectSocket(ApiKeys apiKeys, bool isReconnect) async {
  574. try {
  575. state = AsyncData(state.value!.copyWith(isSocketConnected: false));
  576. await _socket?.disconnect();
  577. _socket = RealtimeSocket();
  578. _socketEventsSubscription = _socket!.events.listen(_onSocketEvent,
  579. onDone: _onSocketDone, onError: _onSocketError);
  580. await _socket?.connect(apiKeys);
  581. await _socket?.events.firstWhere((e) => e is SessionCreatedEvent);
  582. _sendInitialPrompt();
  583. await _socket?.events.firstWhere((e) => e is SessionUpdatedEvent);
  584. if (isReconnect && _lastAudioFromAssistant.isNotEmpty) {
  585. liveLog(() =>
  586. 'Reconnecting, sending last assistant audio to condition voice to match.');
  587. _socket?.sendEvent(
  588. ConversationItemCreateEvent(
  589. item: MessageItem(
  590. id: LiveNotifier.kInjectedStepIdPrefix +
  591. DateTime.now().millisecondsSinceEpoch.toString(),
  592. role: Role.user,
  593. content: [
  594. Content(
  595. type: ContentType.inputAudio,
  596. audio: base64Encode(_lastAudioFromAssistant.takeBytes()),
  597. ),
  598. Content(
  599. type: ContentType.inputText,
  600. text:
  601. 'That audio is the last clip from the assistant before we disconnected, the regional dialect/accent should match that clip. Don\'t mention the reconnection, or re-greet the user. Stay silent, if you have to say something, just note there was a hiccup there.'),
  602. ],
  603. ),
  604. ),
  605. );
  606. await _socket?.events
  607. .firstWhere((e) => e is ConversationItemCreatedEvent);
  608. }
  609. liveLog(() => 'Socket connected.');
  610. final backAudio = _micDataWhenSocketConnectingBuffer
  611. ?.getCurrentSamples()
  612. .buffer
  613. .asUint8List();
  614. if (backAudio != null && backAudio.isNotEmpty) {
  615. liveLog(() =>
  616. 'Sending mic audio while connecting. Duration: ${(backAudio.length / 2.0 / MicrophoneConfig.kSampleRateOpenAiRealtime).toStringAsFixed(3)} s');
  617. final command = MicDataEvent(backAudio);
  618. final result = handleInternalEvent(state.value!, command);
  619. state = AsyncData(result.newState);
  620. _processCommands(result.commands);
  621. } else {
  622. liveLog(() =>
  623. 'No mic audio to send while connecting. _micDataWhenSocketConnectingBuffer: $_micDataWhenSocketConnectingBuffer.');
  624. }
  625. liveLog(() => 'Updating state to indictate socket connected.');
  626. state = AsyncData(state.value!.copyWith(isSocketConnected: true));
  627. } catch (e, s) {
  628. liveLogError(() => 'Error connecting socket: $e\n$s');
  629. state = AsyncData(state.value!.copyWith(isSocketConnected: false));
  630. stop();
  631. }
  632. }
  633.  
  634. void _onSocketDone() {
  635. liveLog(() => 'Socket connection closed by the server or completed');
  636. // If the session is still running, attempt to reconnect.
  637. if (state.value?.isRunning == true) {
  638. liveLog(() => 'Session is running, attempting to reconnect...');
  639. final apiKeys = ref.read(apiKeysProvider);
  640. final result =
  641. handleInternalEvent(state.value!, ReconnectRequestedEvent(apiKeys));
  642. state = AsyncData(result.newState);
  643. _processCommands(result.commands);
  644. } else {
  645. // Not running, just mark socket as disconnected.
  646. state = AsyncData(state.value!.copyWith(isSocketConnected: false));
  647. }
  648. }
  649.  
  650. void _onSocketError(dynamic error, StackTrace stack) {
  651. liveLogError(() => 'Socket error encountered: $error\n$stack');
  652.  
  653. // If we are running, treat this as a lost connection and attempt to reconnect.
  654. if (state.value?.isRunning == true) {
  655. liveLogError(() => 'Socket error while running. Attempting reconnect...');
  656. final apiKeys = ref.read(apiKeysProvider);
  657. final result =
  658. handleInternalEvent(state.value!, ReconnectRequestedEvent(apiKeys));
  659. state = AsyncData(result.newState);
  660. _processCommands(result.commands);
  661. } else {
  662. // Not running, just mark socket as disconnected and log the error.
  663. state = AsyncData(state.value!.copyWith(isSocketConnected: false));
  664. }
  665. }
  666.  
  667. Future<void> _disconnectSocket() async {
  668. await _socket?.disconnect();
  669. await _socketEventsSubscription?.cancel();
  670. _socket = null;
  671. _socketEventsSubscription = null;
  672. }
  673.  
  674. void _startMicrophone(MicrophoneConfig config) {
  675. final micStream = _microphone!.getMicrophoneStream(config);
  676. _microphoneSubscription = micStream.listen((bytes) {
  677. final command = MicDataEvent(bytes);
  678. final result = handleInternalEvent(state.value!, command);
  679. state = AsyncData(result.newState);
  680. _processCommands(result.commands);
  681. }, onError: (e, s) {
  682. liveLogError(() => 'Error in microphone stream: $e\n$s');
  683. }, onDone: () {
  684. liveLog(() => 'Microphone stream done');
  685. });
  686. }
  687.  
  688. Future<void> _stopMicrophone() async {
  689. await _microphone?.stop();
  690. await _microphoneSubscription?.cancel();
  691. _microphone = null;
  692. _microphoneSubscription = null;
  693. }
  694.  
  695. Future<void> _loadPyannoteModel() async {
  696. final modelPath = await getPyannoteModelPath();
  697. _pyannote = Pyannote.load(modelPath);
  698. }
  699.  
  700. void _sendInitialPrompt() async {
  701. Future<String> getInitialPromptWithChatHistory() async {
  702. final initialPrompt = StringBuffer();
  703. initialPrompt
  704. .write(ref.read(settingsNotifierProvider).agent.liveInstructions);
  705. final chatId = state.value?.conversationId;
  706. final String historyString;
  707. if (chatId == null) {
  708. loggy.error('Cannot include history; no conversation ID found');
  709. historyString = '';
  710. } else {
  711. final chat =
  712. ref.read(conversationManagerProvider.notifier).getById(chatId);
  713. if (chat == null) {
  714. loggy.error('Cannot include history; conversation not found');
  715. historyString = '';
  716. } else {
  717. final contextInput = ref.read(companionContextInputProvider);
  718. historyString =
  719. await liveBuildHistoryString(chat, contextInput.tokenCount);
  720. }
  721. }
  722.  
  723. liveLog(() => 'Got chat history string: $historyString');
  724.  
  725. if (historyString.trim().isNotEmpty) {
  726. initialPrompt.write(historyString);
  727. }
  728. return initialPrompt.toString();
  729. }
  730.  
  731. final initialPrompt = await getInitialPromptWithChatHistory();
  732. final updateEvent = SessionUpdateEvent(
  733. session: createDefaultSession(
  734. initialPrompt: initialPrompt,
  735. ref: ref,
  736. ),
  737. );
  738. _socket?.sendEvent(updateEvent);
  739. }
  740.  
  741. static Session createDefaultSession(
  742. {required String initialPrompt, required Ref ref}) {
  743. final scriptId = ref.read(companionScriptIdProvider);
  744. final isAuto = scriptId == null;
  745. final ToolFunction? tool;
  746. if (isAuto) {
  747. tool = selectScriptTool(
  748. '', ref.read(scriptManagerProvider).valueOrNull ?? []);
  749. } else {
  750. final script = ref.read(scriptManagerProvider.notifier).getById(scriptId);
  751. if (script == null) {
  752. liveLogError(() =>
  753. 'script not found for ID: $scriptId. Cannot use tool, will not update session.');
  754. tool = null;
  755. } else {
  756. tool = useScriptTool(script);
  757. }
  758. }
  759.  
  760. final session = Session(
  761. voice: Voice.coral,
  762. instructions: initialPrompt,
  763. temperature: 0.8,
  764. tools: tool == null
  765. ? []
  766. : [tool]
  767. .map(
  768. (t) => Tool(
  769. type: ToolType.function,
  770. name: t.name,
  771. description: t.description,
  772. parameters: jsonDecode(t.parametersAsString),
  773. ),
  774. )
  775. .toList(),
  776. toolChoice: 'auto',
  777. maxResponseOutputTokens: 'inf',
  778. inputAudioTranscription: AudioTranscriptionConfig(model: 'whisper-1'),
  779. modalities: ['text', 'audio'],
  780. inputAudioFormat: AudioFormat.pcm16,
  781. outputAudioFormat: AudioFormat.pcm16,
  782. turnDetection: TurnDetectionConfig(
  783. turnType: 'server_vad',
  784. threshold: 0.5,
  785. prefixPaddingMs: 300,
  786. silenceDurationMs: 1000,
  787. ),
  788. );
  789. return session;
  790. }
  791. }
  792.  
  793.  
  794. // live_internal_handlers.dart
  795. import 'dart:convert';
  796.  
  797. import 'package:telosnex/chat/models/role.dart';
  798. import 'package:telosnex/chat/realtime/realtime_models.dart' hide Role;
  799. import 'package:telosnex/models/loom_step.dart';
  800. import 'package:telosnex/stt/microphone_service.dart';
  801.  
  802. import 'live_architecture.dart';
  803. import 'live_internal_events.dart';
  804. import 'live.dart'; // LiveState defined in live.dart
  805. import 'package:collection/collection.dart';
  806.  
  807. HandlerResult handleInternalEvent(LiveState currentState, InternalEvent event) {
  808. if (event is! MicDataEvent && event is! MicDataWhenRemoteTalkingEvent) {
  809. liveLog(() => 'handleInternalEvent: $event');
  810. }
  811. return switch (event) {
  812. CrosstalkDetectedEvent e => _handleCrosstalkDetected(currentState, e),
  813. RemoteTalkingEndedEvent e => _handleRemoteTalkingEnded(currentState, e),
  814. ScriptChangedEvent e => _handleScriptChanged(currentState, e),
  815. AssistantSpeechStopRequestedEvent e =>
  816. _handleAssistantSpeechStopRequested(currentState, e),
  817. SessionStartRequestedEvent e =>
  818. _handleSessionStartRequested(currentState, e),
  819. LiveStopCalledEvent e => _handleSessionStopRequested(currentState, e),
  820. ReconnectRequestedEvent e => _handleReconnectRequested(currentState, e),
  821. MicDataEvent e => _handleMicDataEvent(currentState, e),
  822. MicDataWhenRemoteTalkingEvent e =>
  823. _handleMicDataWhenRemoteTalking(currentState, e),
  824. MicDataWhenSocketDisconnectedEvent e =>
  825. _handleMicDataWhenSocketDisconnected(currentState, e),
  826. MicMuteEvent e => HandlerResult(currentState.copyWith(isMicMutedByUser: e.isMuted), []),
  827. SpeakerMuteEvent e => HandlerResult(currentState.copyWith(isSpeakerMutedByUser: e.isMuted), []),
  828. };
  829. }
  830.  
  831. HandlerResult _handleMicDataEvent(LiveState state, MicDataEvent event) {
  832. if (!state.isSocketConnected) {
  833. return handleInternalEvent(state, MicDataWhenSocketDisconnectedEvent(event.data));
  834. }
  835.  
  836. if (state.isMicMutedByUser) {
  837. return HandlerResult(state);
  838. }
  839.  
  840. if (state.isRemoteTalking) {
  841. return handleInternalEvent(
  842. state, MicDataWhenRemoteTalkingEvent(event.data));
  843. }
  844. final commands = <DomainCommand>[
  845. SendSocketMessageCommand(
  846. InputAudioBufferAppendEvent(audio: base64Encode(event.data)))
  847. ];
  848. return HandlerResult(state, commands);
  849. }
  850.  
  851. HandlerResult _handleMicDataWhenRemoteTalking(
  852. LiveState state, MicDataWhenRemoteTalkingEvent event) {
  853. if (!state.isRemoteTalking) {
  854. liveLogError(() =>
  855. 'Called _handleMicDataWhenRemoteTalking when not remote talking. Passing to _handleMicDataEvent');
  856. return handleInternalEvent(state, MicDataEvent(event.data));
  857. }
  858. final commands = <DomainCommand>[
  859. BufferMicDataWhenRemoteTalkingCommand(event.data)
  860. ];
  861. return HandlerResult(state, commands);
  862. }
  863.  
  864. HandlerResult _handleMicDataWhenSocketDisconnected(
  865. LiveState state, MicDataWhenSocketDisconnectedEvent event) {
  866. if (state.isSocketConnected) {
  867. liveLogError(() =>
  868. 'Called _handleMicDataWhenSocketDisconnected when socket is connected. Passing to _handleMicDataEvent');
  869. return handleInternalEvent(state, MicDataEvent(event.data));
  870. }
  871. final commands = <DomainCommand>[
  872. BufferMicDataWhenSocketDisconnectedCommand(event.data)
  873. ];
  874. liveLog(() => 'LiveInternalHandlers._handleMicDataWhenSocketDisconnected: '
  875. 'Buffering ${event.data.length} bytes');
  876. return HandlerResult(state, commands);
  877. }
  878.  
  879. HandlerResult _handleCrosstalkDetected(
  880. LiveState state, CrosstalkDetectedEvent event) {
  881. final lastAssistantItemId = state.itemIdToLoomStep.entries
  882. .firstWhereOrNull((e) =>
  883. e.value is LoomChatCompletionText &&
  884. (e.value as LoomChatCompletionText).role == Role.assistant)
  885. ?.key;
  886.  
  887. final commands = <DomainCommand>[];
  888. commands.add(ResetAudioSystemCommand());
  889.  
  890. if (lastAssistantItemId != null) {
  891. commands.add(TruncateAssistantMessageCommand(
  892. lastAssistantItemId, 0, event.audioEndMs));
  893. } else {
  894. liveLogError(
  895. () => '[_handleCrosstalkDetected] No assistant item found to truncate');
  896. }
  897.  
  898. commands.add(ClearRemoteSpeakingMicBufferCommand());
  899.  
  900. if (event.bufferedMicData.isNotEmpty) {
  901. final data = base64Encode(event.bufferedMicData);
  902. commands.add(
  903. SendSocketMessageCommand(InputAudioBufferAppendEvent(audio: data)));
  904. }
  905. final newState = state.copyWith(isRemoteTalking: false);
  906. return HandlerResult(newState, commands);
  907. }
  908.  
  909. HandlerResult _handleRemoteTalkingEnded(
  910. LiveState state, RemoteTalkingEndedEvent _) {
  911. final commands = <DomainCommand>[
  912. ClearRemoteSpeakingMicBufferCommand(),
  913. ];
  914. return HandlerResult(state.copyWith(isRemoteTalking: false), commands);
  915. }
  916.  
  917. HandlerResult _handleScriptChanged(LiveState state, ScriptChangedEvent event) {
  918. return HandlerResult(
  919. state, [RefreshSessionInstructionsCommand(event.scriptId)]);
  920. }
  921.  
  922. HandlerResult _handleAssistantSpeechStopRequested(
  923. LiveState state, AssistantSpeechStopRequestedEvent event) {
  924. final commands = <DomainCommand>[
  925. TruncateAssistantMessageCommand(event.assistantItemId, 0, event.audioEndMs),
  926. ResetAudioSystemCommand(),
  927. ClearRemoteSpeakingMicBufferCommand(),
  928. ];
  929. return HandlerResult(state.copyWith(isRemoteTalking: false), commands);
  930. }
  931.  
  932. HandlerResult _handleSessionStartRequested(
  933. LiveState state, SessionStartRequestedEvent event) {
  934. // Assume apiKeys and pyannotePath are stored in state or known
  935. final apiKeys = event.apiKeys;
  936.  
  937. final commands = <DomainCommand>[
  938. // set conversation *must* come before any commands that may run
  939. // asynchronously. start() calls through here, and expects that
  940. // state.conversationId is set when it runs.
  941. SetConversationCommand(event.chatId),
  942. StartMicrophoneCommand(MicrophoneConfig.openAiRealtime()),
  943. ConnectSocketCommand(apiKeys, false),
  944. LoadPyannoteModelCommand(),
  945. ResetAudioSystemCommand(),
  946. ];
  947. final newState =
  948. state.copyWith(isRunning: true, conversationId: event.chatId);
  949. return HandlerResult(newState, commands);
  950. }
  951.  
  952. HandlerResult _handleSessionStopRequested(
  953. LiveState _, LiveStopCalledEvent __) {
  954. final commands = [
  955. DisconnectSocketCommand(DisconnectReason.remoteDisconnectWithEvent),
  956. StopMicrophoneCommand(),
  957. ResetAudioSystemCommand()
  958. ];
  959.  
  960. return HandlerResult(
  961. LiveState.initial(),
  962. commands);
  963. }
  964.  
  965. HandlerResult _handleReconnectRequested(
  966. LiveState state, ReconnectRequestedEvent event) {
  967. final commands = [
  968. ConnectSocketCommand(event.apiKeys, true),
  969. ];
  970. return HandlerResult(state, commands);
  971. }
  972.  
  973. // live_internal_events.dart
  974. // Internal events for non-server conditions
  975.  
  976. import 'dart:typed_data';
  977.  
  978. import 'package:telosnex/models/api_keys.dart';
  979. import 'package:telosnex/stt/microphone_service.dart';
  980.  
  981. sealed class InternalEvent {}
  982.  
  983. class CrosstalkDetectedEvent extends InternalEvent {
  984. final int audioEndMs;
  985. final Uint8List bufferedMicData;
  986. CrosstalkDetectedEvent(
  987. {required this.audioEndMs, required this.bufferedMicData});
  988. }
  989.  
  990. class MicDataEvent extends InternalEvent {
  991. final Uint8List data;
  992. MicDataEvent(this.data);
  993.  
  994. @override
  995. String toString() =>
  996. 'MicDataEvent(${(data.length / 2.0 / MicrophoneConfig.kSampleRateOpenAiRealtime).toStringAsFixed(3)} seconds, ${data.length} bytes)';
  997. }
  998.  
  999. class MicDataWhenRemoteTalkingEvent extends InternalEvent {
  1000. final Uint8List data;
  1001. MicDataWhenRemoteTalkingEvent(this.data);
  1002. }
  1003.  
  1004. class MicDataWhenSocketDisconnectedEvent extends InternalEvent {
  1005. final Uint8List data;
  1006. MicDataWhenSocketDisconnectedEvent(this.data);
  1007. }
  1008.  
  1009. class RemoteTalkingEndedEvent extends InternalEvent {}
  1010.  
  1011. class ScriptChangedEvent extends InternalEvent {
  1012. final String? scriptId;
  1013. ScriptChangedEvent(this.scriptId);
  1014. }
  1015.  
  1016. class AssistantSpeechStopRequestedEvent extends InternalEvent {
  1017. final String assistantItemId;
  1018. final int audioEndMs;
  1019. AssistantSpeechStopRequestedEvent(this.assistantItemId, this.audioEndMs);
  1020. }
  1021.  
  1022. // Lifecycle events
  1023. class SessionStartRequestedEvent extends InternalEvent {
  1024. final String initialPrompt;
  1025. final String? chatId;
  1026. final ApiKeys apiKeys;
  1027. SessionStartRequestedEvent(this.initialPrompt, this.chatId, this.apiKeys);
  1028. }
  1029.  
  1030. class LiveStopCalledEvent extends InternalEvent {}
  1031.  
  1032. class ReconnectRequestedEvent extends InternalEvent {
  1033. final ApiKeys apiKeys;
  1034. ReconnectRequestedEvent(this.apiKeys);
  1035. }
  1036.  
  1037. class SpeakerMuteEvent extends InternalEvent {
  1038. final bool isMuted;
  1039. SpeakerMuteEvent({required this.isMuted});
  1040. }
  1041.  
  1042. class MicMuteEvent extends InternalEvent {
  1043. final bool isMuted;
  1044. MicMuteEvent({required this.isMuted});
  1045. }
  1046.  
Add Comment
Please, Sign In to add comment