📌 Project State & Architecture Overview

Current State:
We have successfully built a local, hybrid voice assistant bridge. The system consists of a Python orchestrator running on an ODROID-H4 (Intel N97, CPU-only for inference) that connects our local audio hardware to the OpenClaw WebSocket Gateway.

Core Stack:

    Wake Word: openwakeword running locally with a custom-trained .onnx model.

    VAD (Voice Activity Detection): webrtcvad to manage recording states and barge-in.

    STT (Speech-to-Text): faster_whisper (int8, small model) running locally on CPU. We implemented a fallback mechanism: if Whisper's confidence is below 40%, it routes the audio to Google Cloud Speech-to-Text for high-precision parsing.

    TTS (Text-to-Speech): Google Cloud TTS (Wavenet) running asynchronously in a dedicated queue thread.

    UI: A local WebSocket server broadcasts states (idle, listening, thinking, speaking) to a Chromium kiosk running a custom Three.js visualizer (app.js + index.html).

Current Limitations & Roadblocks (Where we need advice):

    STT CPU Latency (The Cold Start): Running faster_whisper on the CPU results in a massive cold-start delay on the first query (sometimes taking up to 19 seconds to transcribe 3 seconds of audio). We are planning to add a "dummy/empty" transcription on startup to warm up the RAM, but we'd love any architectural advice on optimizing local CPU transcription. Should we be streaming chunks to Whisper instead of waiting for the silence_to_stop_ms to trigger?

    Turn-taking / End-of-Speech: We initially had the VAD silence threshold at 3000ms, which felt too slow. We plan to reduce it to ~1200ms, but finding the perfect balance between cutting the user off during a pause and responding snappily is tricky.

Below are the sanitized source files for a final code review.
1. Python Orchestrator (assistant_bridge.py)
Python

#!/usr/bin/env python3
import asyncio, json, time, subprocess, sys, os, queue, threading, collections, enum, unicodedata, re, math
import base64, hashlib
import numpy as np
import sounddevice as sd
import websockets
import webrtcvad
import openwakeword
from openwakeword.model import Model
from faster_whisper import WhisperModel
from nacl.signing import SigningKey

# Cloud Config (Sanitized)
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "/path/to/your/google-key.json"

try:
    from google.cloud import speech
    speech_client = speech.SpeechClient()
    GOOGLE_STT_AVAILABLE = True
except Exception:
    GOOGLE_STT_AVAILABLE = False

try:
    from google.cloud import texttospeech
    tts_client = texttospeech.TextToSpeechClient()
    GOOGLE_TTS_AVAILABLE = True
except Exception:
    GOOGLE_TTS_AVAILABLE = False

GATEWAY_TOKEN = "YOUR_GATEWAY_TOKEN_HERE"
URI = "ws://127.0.0.1:18789"
WORKSPACE_PATH = "/path/to/your/workspace"

# ==========================================
# UI WEBSOCKET BROADCASTER
# ==========================================
UI_CLIENTS = set()
MAIN_LOOP = None

async def ui_broadcast(state_name):
    if UI_CLIENTS:
        msg = json.dumps({"state": state_name})
        await asyncio.gather(*[client.send(msg) for client in UI_CLIENTS], return_exceptions=True)

async def ui_server_handler(websocket):
    UI_CLIENTS.add(websocket)
    try:
        await websocket.wait_closed()
    finally:
        UI_CLIENTS.remove(websocket)

# ==========================================
# AUDIO & SFX ENGINE
# ==========================================
THINKING_PROC = None

def play_sound(action):
    global THINKING_PROC

    ui_map = {
        "wakeup": "listening",
        "thinking": "thinking",
        "stop_thinking": "speaking",
        "idle": "idle",
        "interrupt": "listening",
        "high_precision": "google"
    }

    if action in ui_map:
        global MAIN_LOOP
        if MAIN_LOOP and MAIN_LOOP.is_running():
            asyncio.run_coroutine_threadsafe(ui_broadcast(ui_map[action]), MAIN_LOOP)

    try:
        if action == "wakeup":
            subprocess.Popen(["play", "-q", f"{WORKSPACE_PATH}/wakeup.mp3"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
        elif action == "thinking":
            THINKING_PROC = subprocess.Popen(["play", "-q", f"{WORKSPACE_PATH}/thinking.mp3", "repeat", "999"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
        elif action == "stop_thinking":
            if THINKING_PROC and THINKING_PROC.poll() is None:
                THINKING_PROC.kill()
            THINKING_PROC = None
        elif action == "idle":
            subprocess.Popen(["play", "-q", f"{WORKSPACE_PATH}/idle.mp3", "trim", "0", "1"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
        elif action == "high_precision":
            subprocess.Popen(["play", "-q", f"{WORKSPACE_PATH}/high_precision.mp3"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
        elif action == "error":
            subprocess.Popen(["play", "-q", f"{WORKSPACE_PATH}/error.mp3"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
        elif action == "interrupt":
            subprocess.Popen(["paplay", "/usr/share/sounds/freedesktop/stereo/window-attention.oga"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
    except Exception: pass

# ==========================================
# HELPER FUNCTIONS
# ==========================================
def b64url_no_pad(data: bytes) -> str:
    return base64.urlsafe_b64encode(data).decode('ascii').rstrip('=')

def norm_text(s: str) -> str:
    s = s.lower().strip()
    s = re.sub(r'[^\w\s]', '', s)
    s = unicodedata.normalize("NFKD", s)
    s = "".join(ch for ch in s if not unicodedata.combining(ch))
    return s.strip()

STOP_WORDS = {
    "goodbye", "stop listening", "cancel", "stop", "shut up"
}
STOP_WORDS_NORM = {norm_text(x) for x in STOP_WORDS}

class OpenClawWSClient:
    def __init__(self, uri, token):
        self.uri = uri
        self.token = token
        self.ws = None
        self.req_id = 2
        self.session_key = "agent:main:main"
        self.on_text = None
        self.on_final = None
        self.on_error = None

    async def connect_and_loop(self):
        private_key = SigningKey.generate()
        public_key_bytes = private_key.verify_key.encode()
        device_id = hashlib.sha256(public_key_bytes).hexdigest()

        while True:
            try:
                async with websockets.connect(self.uri) as ws:
                    self.ws = ws
                    msg = await ws.recv()
                    nonce = json.loads(msg)["payload"]["nonce"]
                    signed_at_ms = str(int(time.time() * 1000))

                    sig_string = f"v3|{device_id}|cli|backend|operator|operator.read,operator.write|{signed_at_ms}|{self.token}|{nonce}|linux|desktop"
                    sig_str = b64url_no_pad(private_key.sign(sig_string.encode('utf-8')).signature)

                    auth_req = {
                        "type": "req", "id": "auth-1", "method": "connect",
                        "params": {
                            "minProtocol": 3, "maxProtocol": 3,
                            "client": {"id": "cli", "version": "1.0.0", "platform": "linux", "deviceFamily": "desktop", "mode": "backend"},
                            "role": "operator", "scopes": ["operator.read", "operator.write"],
                            "auth": {"token": self.token},
                            "device": {"id": device_id, "publicKey": b64url_no_pad(public_key_bytes), "signature": sig_str, "signedAt": int(signed_at_ms), "nonce": nonce}
                        }
                    }
                    await ws.send(json.dumps(auth_req))
                    await ws.recv()
                    print("✅ Native Brain connected to Gateway!")

                    while True:
                        msg = await ws.recv()
                        obj = json.loads(msg)
                        if obj.get("event") == "agent":
                            stream = obj.get("payload", {}).get("stream")
                            data = obj.get("payload", {}).get("data", {})
                            if stream == "assistant" and "delta" in data:
                                if self.on_text: self.on_text(data["delta"])
                            elif stream == "lifecycle" and data.get("phase") == "end":
                                if self.on_final: self.on_final("")
            except Exception as e:
                print(f"\n❌ Connection lost. Reconnecting in 3s... ({e})")
                play_sound("error")
                if self.on_error: self.on_error(str(e))
                await asyncio.sleep(3)

    async def send_chat(self, text):
        req = {"type": "req", "id": f"req-{self.req_id}", "method": "chat.send",
               "params": {"sessionKey": self.session_key, "message": text, "idempotencyKey": f"msg-{self.req_id}-{int(time.time()*1000)}"}}
        self.req_id += 1
        await self.ws.send(json.dumps(req))

    async def abort_current(self):
        req = {"type": "req", "id": f"req-{self.req_id}", "method": "chat.abort", "params": {"sessionKey": self.session_key}}
        self.req_id += 1
        await self.ws.send(json.dumps(req))

class SentenceChunker:
    def __init__(self, on_chunk):
        self.buf = ""
        self.on_chunk = on_chunk

    def push(self, text_delta: str):
        self.buf += text_delta
        while True:
            cut = -1
            for marker in [". ", "? ", "! ", "\n", ".\n"]:
                idx = self.buf.find(marker)
                if idx != -1:
                    cut = idx + len(marker)
                    break
            if cut == -1:
                if len(self.buf.split()) >= 15:
                    chunk = self.buf.strip()
                    self.buf = ""
                    if chunk: self.on_chunk(chunk)
                break
            chunk = self.buf[:cut].strip()
            self.buf = self.buf[cut:].lstrip()
            if chunk: self.on_chunk(chunk)

    def flush(self):
        tail = self.buf.strip()
        self.buf = ""
        if tail: self.on_chunk(tail)

    def reset(self):
        self.buf = ""

# ==========================================
# PREMIUM TTS ENGINE (GOOGLE CLOUD)
# ==========================================
class GoogleTTSThread(threading.Thread):
    def __init__(self):
        super().__init__(daemon=True)
        self.q = queue.Queue()
        self.idle = threading.Event()
        self.idle.set()
        self.player = None
        self.lock = threading.Lock()
        self.abort_flag = False
        self.is_speaking_now = False

    def enqueue(self, text: str):
        text = text.strip()
        if text:
            self.idle.clear()
            self.q.put(text)

    def stop_now(self):
        self.abort_flag = True
        try:
            while True: self.q.get_nowait()
        except queue.Empty: pass
        with self.lock:
            if self.player and self.player.poll() is None:
                self.player.kill()
        self.idle.set()
        self.is_speaking_now = False

    def wait_until_idle(self):
        while not self.idle.is_set(): time.sleep(0.05)

    def run(self):
        while True:
            text = self.q.get()
            if text is None: break
            self.abort_flag = False

            if self.abort_flag:
                if self.q.empty(): self.idle.set()
                continue

            try:
                synthesis_input = texttospeech.SynthesisInput(text=text)
                voice = texttospeech.VoiceSelectionParams(
                    language_code="en-US", name="en-US-Wavenet-F"
                )
                audio_config = texttospeech.AudioConfig(
                    audio_encoding=texttospeech.AudioEncoding.LINEAR16,
                    sample_rate_hertz=24000
                )

                response = tts_client.synthesize_speech(
                    input=synthesis_input, voice=voice, audio_config=audio_config
                )

                if self.abort_flag:
                    if self.q.empty(): self.idle.set()
                    continue

                with self.lock:
                    self.player = subprocess.Popen(
                        ["paplay", "--raw", "--format=s16le", "--channels=1", "--rate", "24000"],
                        stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
                    )

                self.is_speaking_now = True
                self.player.communicate(input=response.audio_content)
                self.is_speaking_now = False

            except Exception as e:
                print(f"\n⚠️ [Cloud TTS Error]: {e}")
                self.is_speaking_now = False
                play_sound("error")

            finally:
                if self.q.empty():
                    self.idle.set()

class State(enum.Enum):
    IDLE = "idle"; RECORDING = "recording"; THINKING = "thinking"; SPEAKING = "speaking"; FOLLOWUP = "followup"

class VoiceLoop:
    def __init__(self, tts_engine):
        self.sample_rate = 16000
        self.frame_ms = 20
        self.audio_q = queue.Queue(maxsize=256)
        self.tts = tts_engine

        self.wakeword_file = f"{WORKSPACE_PATH}/custom_wakeword.onnx"

        self.oww = Model(wakeword_model_paths=[self.wakeword_file])
        self.wake_word_key = list(self.oww.models.keys())[0]
        self.vad = webrtcvad.Vad(3)

        self.state = State.IDLE
        self.oww_batch = collections.deque(maxlen=4)
        self.preroll = collections.deque(maxlen=40)
        self.capture_frames = []

        self.last_speech_ts = 0.0
        self.rearm_until = 0.0
        self.followup_deadline = 0.0
        self.silence_to_stop_ms = 3000
        self.min_utterance_ms = 400

        self.continuous_mode = True
        self.followup_timeout_s = 5.0

        self.barge_frames_needed = 20
        self.barge_speech_frames = 0

        self.on_utterance_ready = None
        self.on_barge_in = None

    def audio_callback(self, indata, frames, time_info, status):
        if status: play_sound("error")
        mono = indata[:, 0]
        pcm16 = (np.clip(mono, -1.0, 1.0) * 32767).astype(np.int16).tobytes()
        try: self.audio_q.put_nowait(pcm16)
        except queue.Full: pass

    def drain_audio_q(self):
        try:
            while True: self.audio_q.get_nowait()
        except queue.Empty: pass

    def rearm_idle(self, cooldown_s=1.0, play_idle_sound=False):
        play_sound("stop_thinking")
        if play_idle_sound or self.state == State.FOLLOWUP:
            play_sound("idle")

        self.drain_audio_q()
        self.oww = Model(wakeword_model_paths=[self.wakeword_file])
        self.wake_word_key = list(self.oww.models.keys())[0]
        self.oww_batch.clear()
        self.preroll.clear()
        self.capture_frames.clear()
        self.last_speech_ts = 0.0
        self.barge_speech_frames = 0
        self.followup_deadline = 0.0
        self.rearm_until = time.monotonic() + cooldown_s
        self.set_state(State.IDLE)

    def set_state(self, new_state: State):
        if new_state != self.state:
            print(f"\n🔄 [{self.state.value.upper()} -> {new_state.value.upper()}]")
            self.state = new_state

            ui_state = "idle"
            if new_state == State.RECORDING or new_state == State.FOLLOWUP: ui_state = "listening"
            elif new_state == State.THINKING: ui_state = "thinking"
            elif new_state == State.SPEAKING: ui_state = "speaking"

            global MAIN_LOOP
            if MAIN_LOOP and MAIN_LOOP.is_running():
                asyncio.run_coroutine_threadsafe(ui_broadcast(ui_state), MAIN_LOOP)

    def begin_recording(self, now: float):
        play_sound("wakeup")
        self.capture_frames = list(self.preroll)
        self.last_speech_ts = now
        self.barge_speech_frames = 0
        self.set_state(State.RECORDING)

    def after_speaking_done(self):
        if self.continuous_mode:
            self.drain_audio_q()
            self.oww_batch.clear()
            self.preroll.clear()
            self.capture_frames.clear()
            self.barge_speech_frames = 0
            self.followup_deadline = time.monotonic() + self.followup_timeout_s
            self.set_state(State.FOLLOWUP)
        else:
            self.rearm_idle()

    def finish_recording(self):
        audio_bytes = b"".join(self.capture_frames)
        self.capture_frames.clear()
        self.set_state(State.THINKING)
        if self.on_utterance_ready: self.on_utterance_ready(audio_bytes)

    def process_frame(self, frame_bytes):
        now = time.monotonic()

        if self.tts.is_speaking_now:
            self.barge_speech_frames = 0
            return

        is_speech = self.vad.is_speech(frame_bytes, 16000)

        if self.state in (State.IDLE, State.RECORDING, State.THINKING, State.SPEAKING, State.FOLLOWUP):
            self.preroll.append(frame_bytes)

        if self.state in (State.THINKING, State.SPEAKING):
            if is_speech: self.barge_speech_frames += 1
            else: self.barge_speech_frames = 0

            if self.barge_speech_frames >= self.barge_frames_needed:
                if self.on_barge_in: self.on_barge_in()
                self.begin_recording(now)
            return

        if self.state == State.FOLLOWUP:
            if now >= self.followup_deadline:
                print("⏳ 5s timeout reached. Going to sleep...")
                self.rearm_idle()
                return
            if is_speech: self.begin_recording(now)
            return

        if self.state == State.IDLE and now < self.rearm_until: return

        if self.state == State.RECORDING:
            self.capture_frames.append(frame_bytes)
            if is_speech: self.last_speech_ts = now
            silence_ms = (now - self.last_speech_ts) * 1000.0
            sys.stdout.write(f"\r🤫 Silence: {silence_ms/1000:.1f}s / {self.silence_to_stop_ms/1000}s   ")
            sys.stdout.flush()
            utterance_ms = len(self.capture_frames) * self.frame_ms
            if utterance_ms >= self.min_utterance_ms and silence_ms >= self.silence_to_stop_ms:
                self.finish_recording()
            return

        if self.state == State.IDLE:
            self.oww_batch.append(frame_bytes)
            if len(self.oww_batch) < 4: return
            chunk_80ms = b"".join(self.oww_batch)
            self.oww_batch.clear()
            if self.oww.predict(np.frombuffer(chunk_80ms, dtype=np.int16)).get(self.wake_word_key, 0.0) >= 0.10:
                print(f"\n🔔 Wake Word detected!")
                self.begin_recording(now)

    def run(self):
        try:
            with sd.InputStream(samplerate=16000, channels=1, dtype="float32", blocksize=320, callback=self.audio_callback):
                print("🎧 Assistant listening for Wake Word...")
                while True: self.process_frame(self.audio_q.get())
        except Exception as e:
            print(f"❌ Fatal mic error: {e}")
            play_sound("error")

class AssistantApp:
    def __init__(self):
        print("\n⚙️ Initializing AI modules...")
        self.whisper = WhisperModel("small", device="cpu", compute_type="int8")

        if GOOGLE_STT_AVAILABLE:
            print("☁️ Cloud Connection (Backup Ears): Ready")
        else:
            print("💻 Cloud Connection (Ears): Inactive.")

        if GOOGLE_TTS_AVAILABLE:
            print("☁️ Cloud Connection (Premium Voice): Ready")
        else:
            print("💻 ERROR: Google TTS unavailable. Check keys!")

        self.ws_loop = asyncio.new_event_loop()
        global MAIN_LOOP
        MAIN_LOOP = self.ws_loop

        self.ws = OpenClawWSClient(URI, GATEWAY_TOKEN)

        self.tts = GoogleTTSThread()
        self.chunker = SentenceChunker(self.tts.enqueue)

        self.voice = VoiceLoop(self.tts)

        self.run_finished = threading.Event()
        self.run_finished.set()

        self.current_turn_text = ""
        self.t_start_ws = 0.0
        self.first_word = False

        self.voice.on_utterance_ready = self.handle_utterance
        self.voice.on_barge_in = self.handle_barge_in
        self.ws.on_text = self.handle_llm_delta
        self.ws.on_final = self.handle_llm_final
        self.ws.on_error = self.handle_llm_error

    def start(self):
        html_path = f"{WORKSPACE_PATH}/voiceorb/index.html"
        if not os.path.exists(html_path):
            print(f"\n❌ ERROR: File not found at: {html_path}")
        else:
            print("🌐 Launching Assistant UI...")
            subprocess.Popen(["chromium", "--kiosk", f"file://{html_path}"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)

        threading.Thread(target=self._start_ws, daemon=True).start()
        self.tts.start()
        time.sleep(1)
        self.voice.run()

    async def run_all_services(self):
        print("📺 Visual Server activated on port 8765")
        async with websockets.serve(ui_server_handler, "127.0.0.1", 8765):
            await self.ws.connect_and_loop()

    def _start_ws(self):
        asyncio.set_event_loop(self.ws_loop)
        self.ws_loop.run_until_complete(self.run_all_services())

    def _run_ws(self, coro, timeout=10.0):
        try:
            future = asyncio.run_coroutine_threadsafe(coro, self.ws_loop)
            return future.result(timeout=timeout)
        except Exception as e:
            raise RuntimeError(f"Timeout or Gateway failure: {e}")

    def handle_utterance(self, audio_bytes):
        play_sound("thinking")
        threading.Thread(target=self._transcribe, args=(audio_bytes,), daemon=True).start()

    def _transcribe(self, audio_bytes):
        t_start_stt = time.time()
        audio = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32) / 32768.0
        segs, info = self.whisper.transcribe(audio, language="en")
        segments = list(segs)
        text = " ".join(s.text.strip() for s in segments).strip()

        if segments:
            avg_prob = sum(math.exp(s.avg_logprob) for s in segments) / len(segments)
            confidence = avg_prob * 100
        else:
            confidence = 0.0

        print(f"\n   ⏱️ [STT Whisper] ({time.time() - t_start_stt:.2f}s) | Confidence: {confidence:.1f}%")

        if not text:
            play_sound("stop_thinking")
            self.voice.rearm_idle()
            return

        text_norm = norm_text(text)
        final_engine = "Local Whisper"

        if GOOGLE_STT_AVAILABLE:
            needs_help = confidence < 40.0
            if needs_help:
                play_sound("high_precision")
                try:
                    audio_google = speech.RecognitionAudio(content=audio_bytes)
                    config = speech.RecognitionConfig(
                        encoding=speech.RecognitionConfig.AudioEncoding.LINEAR16,
                        sample_rate_hertz=16000, language_code="en-US",
                    )
                    response = speech_client.recognize(config=config, audio=audio_google)
                    if response.results:
                        text = response.results[0].alternatives[0].transcript
                        text_norm = norm_text(text)
                        final_engine = "Google Cloud"
                except Exception:
                    play_sound("error")

        print(f"👤 User ({final_engine}): {text}")

        if self.voice.state == State.IDLE:
            play_sound("stop_thinking")
            return

        if any(word in text_norm for word in STOP_WORDS_NORM):
            play_sound("stop_thinking")
            self.voice.rearm_idle(play_idle_sound=True)
            return

        self.run_finished.clear()
        self.current_turn_text = ""
        self.first_word = False
        self.voice.set_state(State.THINKING)

        self.t_start_ws = time.time()
        try:
            self._run_ws(self.ws.send_chat(text))
        except Exception as e:
            print(f"[Connection Error] {e}")
            play_sound("error")
            play_sound("stop_thinking")
            self.run_finished.set()
            self.voice.rearm_idle()

    def handle_llm_delta(self, text_delta):
        if not self.first_word:
            play_sound("stop_thinking")
            print(f"   ⏱️ [TTFT]: {time.time() - self.t_start_ws:.2f}s")
            self.first_word = True
        if self.voice.state == State.THINKING:
            self.voice.set_state(State.SPEAKING)
        sys.stdout.write(text_delta); sys.stdout.flush()
        self.current_turn_text += text_delta
        self.chunker.push(text_delta)

    def handle_llm_final(self, full_text):
        print("\n")
        play_sound("stop_thinking")
        self.chunker.flush()
        self.run_finished.set()
        threading.Thread(target=self._wait_tts, daemon=True).start()

    def handle_llm_error(self, err):
        play_sound("error")
        play_sound("stop_thinking")
        self.chunker.reset(); self.tts.stop_now(); self.run_finished.set(); self.voice.rearm_idle()

    def _wait_tts(self):
        self.tts.wait_until_idle()
        if self.run_finished.is_set() and self.voice.state in (State.THINKING, State.SPEAKING):
            self.voice.after_speaking_done()

    def handle_barge_in(self):
        play_sound("stop_thinking")
        play_sound("interrupt")
        print("\n🛑 INTERRUPTION DETECTED! Canceling audio...")
        self.chunker.reset()
        self.tts.stop_now()
        try: self._run_ws(self.ws.abort_current())
        except Exception: pass
        self.run_finished.set()

if __name__ == "__main__":
    import warnings
    warnings.filterwarnings("ignore", category=UserWarning)
    try: AssistantApp().start()
    except KeyboardInterrupt: print("\nSee you later!")

2. Three.js Visualizer (app.js)
JavaScript

let scene, camera, renderer, orbLayers = [], currentState = 'idle';
let targetScale = 1.0, currentScale = 1.0;

// Color and State Configurations
const states = {
    idle: { // Pale and smaller
        layers: [
            { color: 0x2a2e5e, opacity: 0.1, scale: 0.7, rotationSpeed: { x: 0.0005, y: 0.001 } },
            { color: 0x2a2e5e, opacity: 0.1, scale: 0.6, rotationSpeed: { x: -0.001, y: 0.0015 } }
        ],
        timeSpeed: 0.008, pulsate: false, chromaticAberration: 0.2, description: 'Sleeping'
    },
    listening: {
        layers: [
            { color: 0x434FCF, opacity: 0.2, scale: 1.0, rotationSpeed: { x: 0.002, y: 0.004 } },
            { color: 0x434FCF, opacity: 0.4, scale: 0.7, rotationSpeed: { x: 0.004, y: -0.003 } }
        ],
        timeSpeed: 0.022, pulsate: true, pulsateMin: 0.02, pulsateMax: 0.2, chromaticAberration: 1.2, description: 'Listening'
    },
    google: { // Visual bridge: Electric Cyan for High Precision parsing
        layers: [
            { color: 0x00f2ff, opacity: 0.3, scale: 1.1, rotationSpeed: { x: 0.008, y: 0.008 } },
            { color: 0x00f2ff, opacity: 0.5, scale: 0.8, rotationSpeed: { x: -0.01, y: 0.01 } }
        ],
        timeSpeed: 0.05, pulsate: true, pulsateMin: 0.1, pulsateMax: 0.4, chromaticAberration: 2.5, description: 'High Precision'
    },
    thinking: {
        layers: [
            { color: 0x8747F7, opacity: 0.2, scale: 0.85, rotationSpeed: { x: 0.003, y: 0.003 } },
            { color: 0x8747F7, opacity: 0.4, scale: 0.60, rotationSpeed: { x: 0.005, y: -0.004 } }
        ],
        timeSpeed: 0.02, pulsate: true, pulsateMin: 0.0, pulsateMax: 0.15, chromaticAberration: 0.8, description: 'Thinking'
    },
    speaking: {
        layers: [
            { color: 0xFF1893, opacity: 0.2, scale: 1.0, rotationSpeed: { x: 0.004, y: 0.005 } },
            { color: 0xFF1893, opacity: 0.4, scale: 0.70, rotationSpeed: { x: 0.006, y: -0.005 } }
        ],
        timeSpeed: 0.027, pulsate: true, pulsateMin: 0.05, pulsateMax: 0.22, chromaticAberration: 1.5, description: 'Speaking'
    }
};

const vertexShader = `
    varying vec3 vNormal;
    varying vec3 vPosition;
    uniform float time;
    uniform float audioLevel;
    void main() {
        vNormal = normalize(normalMatrix * normal);
        vec3 pos = position;
        float distortion = sin(pos.y * 3.0 + time) * 0.02 * (1.0 + audioLevel);
        pos = pos + normal * distortion;
        vPosition = pos;
        gl_Position = projectionMatrix * modelViewMatrix * vec4(pos, 1.0);
    }
`;

const fragmentShader = `
    varying vec3 vNormal;
    varying vec3 vPosition;
    uniform vec3 sphereColor;
    uniform float opacity;
    uniform float chromaticAberration;
    void main() {
        vec3 viewDirection = normalize(cameraPosition - vPosition);
        float fresnel = pow(1.0 - abs(dot(viewDirection, normalize(vNormal))), 2.0);
        vec3 color = sphereColor + (fresnel * chromaticAberration * 0.3);
        gl_FragColor = vec4(color, opacity + (fresnel * 0.4));
    }
`;

function init() {
    scene = new THREE.Scene();
    camera = new THREE.PerspectiveCamera(75, window.innerWidth / window.innerHeight, 0.1, 1000);
    camera.position.z = 5;
    const canvas = document.getElementById('canvas');
    renderer = new THREE.WebGLRenderer({ canvas, antialias: true, alpha: true });
    renderer.setSize(window.innerWidth, window.innerHeight);

    // Create 2 fixed layers to allow smooth transitions
    for (let i = 0; i < 2; i++) {
        const geometry = new THREE.SphereGeometry(1, 64, 64);
        const material = new THREE.ShaderMaterial({
            vertexShader, fragmentShader,
            uniforms: {
                time: { value: 0 }, audioLevel: { value: 0 },
                sphereColor: { value: new THREE.Color(0x000000) },
                opacity: { value: 0 }, chromaticAberration: { value: 0 }
            },
            transparent: true, depthWrite: false
        });
        const sphere = new THREE.Mesh(geometry, material);
        sphere.userData = { currentScale: 0.1, targetScale: 0.1, rot: { x: 0, y: 0 } };
        scene.add(sphere);
        orbLayers.push(sphere);
    }

    window.addEventListener('resize', () => {
        camera.aspect = window.innerWidth / window.innerHeight;
        camera.updateProjectionMatrix();
        renderer.setSize(window.innerWidth, window.innerHeight);
    });
    animate();
}

function setState(stateName) {
    if (!states[stateName]) return;
    currentState = stateName;
    const statusEl = document.getElementById('status');
    if (statusEl) {
        statusEl.textContent = states[stateName].description;
    }
}

function animate() {
    requestAnimationFrame(animate);
    const state = states[currentState];
    const lerpSpeed = 0.05; // Controls the smoothness of the transition

    orbLayers.forEach((layer, index) => {
        const config = state.layers[index] || { color: 0x000000, opacity: 0, scale: 0, rotationSpeed: { x: 0, y: 0 } };
        const uniforms = layer.material.uniforms;

        // Smooth Color Interpolation
        uniforms.sphereColor.value.lerp(new THREE.Color(config.color), lerpSpeed);

        // Smooth Opacity and Aberration Interpolation
        uniforms.opacity.value += (config.opacity - uniforms.opacity.value) * lerpSpeed;
        uniforms.chromaticAberration.value += (state.chromaticAberration - uniforms.chromaticAberration.value) * lerpSpeed;

        // Smooth Scale Interpolation
        let pulse = state.pulsate ? (Math.sin(Date.now() * 0.005) * (state.pulsateMax - state.pulsateMin)) : 0;
        let tScale = config.scale + pulse;
        layer.scale.setScalar(layer.scale.x + (tScale - layer.scale.x) * lerpSpeed);

        // Animation
        uniforms.time.value += state.timeSpeed;
        layer.rotation.x += config.rotationSpeed.x;
        layer.rotation.y += config.rotationSpeed.y;
    });

    renderer.render(scene, camera);
}

function connectToAssistant() {
    const socket = new WebSocket('ws://127.0.0.1:8765');

    socket.onopen = () => {
        const statusEl = document.getElementById('status');
        if (statusEl) statusEl.textContent = 'Assistant Online';
    };

    socket.onmessage = (event) => {
        try {
            const data = JSON.parse(event.data);
            if (data.state) setState(data.state);
        } catch (e) {}
    };
    socket.onclose = () => setTimeout(connectToAssistant, 2000);
}

window.addEventListener('DOMContentLoaded', () => {
    init();
    connectToAssistant();
});

3. Visual UI Base (index.html)
HTML

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Voice Assistant UI - Smooth</title>
    <link rel="stylesheet" href="style.css">
    <style>
        body {
            cursor: none;
            background-color: #000;
            margin: 0;
            overflow: hidden;
        }
        #controls {
            pointer-events: none;
            border: none;
            background: none;
        }
        .state-buttons, #startBtn, .theme-toggle {
            display: none !important;
        }
        #status {
            bottom: 30px;
            opacity: 0.3;
            font-size: 12px;
            letter-spacing: 2px;
            text-transform: uppercase;
            color: white;
            position: fixed;
            width: 100%;
            text-align: center;
        }
    </style>
</head>
<body>
    <div id="container">
        <canvas id="canvas"></canvas>
        <div id="controls">
            <div id="status">Connecting...</div>
        </div>
    </div>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>
    <script src="app.js"></script>
</body>
</html>