From e1ef5bd90f0f5aa3ec63067636b0b242df7b37ae Mon Sep 17 00:00:00 2001 From: Baskaran Balasubramanian Date: Fri, 10 Apr 2026 04:18:57 +0530 Subject: [PATCH 01/10] add live voice session scaffold --- apps/api-edge/src/routes/voice.ts | 198 ++++++++++++++++++++ apps/api-edge/src/types.ts | 10 ++ apps/meridian/app.json | 12 ++ apps/meridian/app/(main)/converse.tsx | 249 ++++++++++++++++++++++++-- apps/meridian/app/_layout.tsx | 1 + apps/meridian/lib/api.ts | 43 +++++ apps/meridian/lib/liveVoice.ts | 119 ++++++++++++ apps/meridian/lib/livekitGlobals.ts | 11 ++ apps/meridian/lib/telemetry.ts | 4 + apps/meridian/lib/voiceSession.ts | 42 +++++ apps/meridian/package.json | 5 + 11 files changed, 678 insertions(+), 16 deletions(-) create mode 100644 apps/meridian/lib/liveVoice.ts create mode 100644 apps/meridian/lib/livekitGlobals.ts create mode 100644 apps/meridian/lib/voiceSession.ts diff --git a/apps/api-edge/src/routes/voice.ts b/apps/api-edge/src/routes/voice.ts index c11155b..6c0fa73 100644 --- a/apps/api-edge/src/routes/voice.ts +++ b/apps/api-edge/src/routes/voice.ts @@ -17,6 +17,138 @@ export const voiceRouter = new Hono<{ Bindings: Env; Variables: Variables }>(); // Daniel (onwK4e9ZLuTAKqWW03F9) — deeper/more formal, but noticeably slower to first byte. const DEFAULT_ELEVENLABS_VOICE_ID = 'JBFqnCBsd6RMkjVDRZzb'; +type VoiceSessionMode = 'batch' | 'live'; +type VoiceSessionProvider = 'batch_proxy' | 'livekit' | 'openai_realtime'; + +type VoiceSessionConnectResponse = { + mode: 'live'; + provider: 'livekit'; + serverUrl: string; + roomName: string; + participantIdentity: string; + participantToken: string; + expiresAt: string; + planningToolsAvailableDuringConversation: true; + bookingToolsLockedUntilConfirm: true; +}; + +function getLiveVoiceProvider(env: Env): VoiceSessionProvider | null { + if (env.LIVEKIT_URL && env.LIVEKIT_API_KEY && env.LIVEKIT_API_SECRET) { + return 'livekit'; + } + if (env.OPENAI_API_KEY && env.OPENAI_REALTIME_MODEL) { + return 'openai_realtime'; + } + return null; +} + +function base64UrlEncode(input: string | Uint8Array): string { + const bytes = typeof input === 'string' ? new TextEncoder().encode(input) : input; + let binary = ''; + for (const byte of bytes) binary += String.fromCharCode(byte); + return btoa(binary).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/g, ''); +} + +async function signHs256(secret: string, data: string): Promise { + const key = await crypto.subtle.importKey( + 'raw', + new TextEncoder().encode(secret), + { name: 'HMAC', hash: 'SHA-256' }, + false, + ['sign'], + ); + const signature = await crypto.subtle.sign('HMAC', key, new TextEncoder().encode(data)); + return base64UrlEncode(new Uint8Array(signature)); +} + +function slugPart(value: string, fallback: string): string { + const normalized = value.trim().toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, ''); + return normalized.slice(0, 32) || fallback; +} + +async function createLiveKitParticipantToken(params: { + apiKey: string; + apiSecret: string; + roomName: string; + participantIdentity: string; + metadata?: Record; + ttlSeconds?: number; +}): Promise<{ token: string; expiresAt: string }> { + const now = Math.floor(Date.now() / 1000); + const ttlSeconds = params.ttlSeconds ?? 60 * 60; + const exp = now + ttlSeconds; + const header = { alg: 'HS256', typ: 'JWT' }; + const payload = { + iss: params.apiKey, + sub: params.participantIdentity, + nbf: now - 10, + exp, + video: { + room: params.roomName, + roomJoin: true, + canPublish: true, + canSubscribe: true, + canPublishData: true, + }, + metadata: params.metadata ? JSON.stringify(params.metadata) : undefined, + }; + const encodedHeader = base64UrlEncode(JSON.stringify(header)); + const encodedPayload = base64UrlEncode(JSON.stringify(payload)); + const signature = await signHs256(params.apiSecret, `${encodedHeader}.${encodedPayload}`); + return { + token: `${encodedHeader}.${encodedPayload}.${signature}`, + expiresAt: new Date(exp * 1000).toISOString(), + }; +} + +function getVoiceSessionManifest(env: Env): { + mode: VoiceSessionMode; + transport: 'http' | 'webrtc'; + provider: VoiceSessionProvider; + ready: boolean; + planningToolsAvailableDuringConversation: boolean; + bookingToolsLockedUntilConfirm: boolean; + supportsInterruptions: boolean; + supportsServerVad: boolean; + premiumVoice: 'elevenlabs' | 'system' | 'none'; + fallback: { stt: 'whisper_proxy'; tts: 'elevenlabs_http' | 'none' }; + diagnostics: string[]; +} { + const liveProvider = getLiveVoiceProvider(env); + const diagnostics: string[] = []; + + if (!liveProvider) { + diagnostics.push('live_runtime_not_configured'); + if (!env.LIVEKIT_URL || !env.LIVEKIT_API_KEY || !env.LIVEKIT_API_SECRET) { + diagnostics.push('livekit_credentials_missing'); + } + if (!env.OPENAI_REALTIME_MODEL) { + diagnostics.push('openai_realtime_model_missing'); + } + } + + if (!env.ELEVENLABS_API_KEY) { + diagnostics.push('elevenlabs_unavailable'); + } + + return { + mode: liveProvider ? 'live' : 'batch', + transport: liveProvider ? 'webrtc' : 'http', + provider: liveProvider ?? 'batch_proxy', + ready: !!liveProvider, + planningToolsAvailableDuringConversation: true, + bookingToolsLockedUntilConfirm: true, + supportsInterruptions: !!liveProvider, + supportsServerVad: !!liveProvider, + premiumVoice: env.ELEVENLABS_API_KEY ? 'elevenlabs' : 'none', + fallback: { + stt: 'whisper_proxy', + tts: env.ELEVENLABS_API_KEY ? 'elevenlabs_http' : 'none', + }, + diagnostics, + }; +} + function broLog(event: string, data: Record) { console.log( JSON.stringify({ @@ -87,6 +219,72 @@ function isCfWhisperHallucination(text: string): boolean { return false; } +voiceRouter.get('/session', async (c) => { + const unauthorized = requireBroClientKey(c); + if (unauthorized) return unauthorized; + + return c.json(getVoiceSessionManifest(c.env)); +}); + +voiceRouter.post('/session/connect', async (c) => { + const unauthorized = requireBroClientKey(c); + if (unauthorized) return unauthorized; + + const { LIVEKIT_URL, LIVEKIT_API_KEY, LIVEKIT_API_SECRET } = c.env; + if (!LIVEKIT_URL || !LIVEKIT_API_KEY || !LIVEKIT_API_SECRET) { + return c.json({ error: 'Live voice session is not configured' }, 503); + } + + let body: { hirerId?: string; sessionId?: string }; + try { + body = await c.req.json(); + } catch { + return c.json({ error: 'Invalid JSON body' }, 400); + } + + const hirerId = body.hirerId?.trim(); + if (!hirerId) { + return c.json({ error: 'hirerId required' }, 400); + } + + const roomName = `ace-${slugPart(hirerId, 'guest')}-${slugPart(body.sessionId ?? Date.now().toString(36), 'session')}`; + const participantIdentity = `traveler-${slugPart(hirerId, 'guest')}`; + const { token, expiresAt } = await createLiveKitParticipantToken({ + apiKey: LIVEKIT_API_KEY, + apiSecret: LIVEKIT_API_SECRET, + roomName, + participantIdentity, + metadata: { + hirerId, + sessionId: body.sessionId ?? null, + role: 'traveler', + planningToolsAvailableDuringConversation: true, + bookingToolsLockedUntilConfirm: true, + }, + }); + + const response: VoiceSessionConnectResponse = { + mode: 'live', + provider: 'livekit', + serverUrl: LIVEKIT_URL, + roomName, + participantIdentity, + participantToken: token, + expiresAt, + planningToolsAvailableDuringConversation: true, + bookingToolsLockedUntilConfirm: true, + }; + + broLog('live_session_token_issued', { + hirerId: hirerId.slice(0, 12), + roomName, + participantIdentity, + expiresAt, + }); + + return c.json(response); +}); + voiceRouter.post('/transcribe', async (c) => { const unauthorized = requireBroClientKey(c); if (unauthorized) return unauthorized; diff --git a/apps/api-edge/src/types.ts b/apps/api-edge/src/types.ts index aeab141..11bcd77 100644 --- a/apps/api-edge/src/types.ts +++ b/apps/api-edge/src/types.ts @@ -214,6 +214,16 @@ export interface Env { OPENAI_API_KEY?: string; /** ElevenLabs API key — premium server-side TTS for Ace voice replies. */ ELEVENLABS_API_KEY?: string; + /** LiveKit websocket URL for persistent WebRTC voice sessions. */ + LIVEKIT_URL?: string; + /** LiveKit API key used to mint session access tokens. */ + LIVEKIT_API_KEY?: string; + /** LiveKit API secret paired with LIVEKIT_API_KEY. */ + LIVEKIT_API_SECRET?: string; + /** Optional OpenAI Realtime model name for live speech sessions. */ + OPENAI_REALTIME_MODEL?: string; + /** Optional ElevenLabs voice/agent identifier for live voice rendering. */ + ELEVENLABS_VOICE_ID?: string; /** Google Gemini API key — opt-in paid tier for high-volume extraction. Get at aistudio.google.com. Enable billing to remove RPD limits. */ GEMINI_API_KEY?: string; /** Firecrawl API key — enables markdown scraping for operators without first-party APIs. */ diff --git a/apps/meridian/app.json b/apps/meridian/app.json index 33ac7ef..dc44bd9 100644 --- a/apps/meridian/app.json +++ b/apps/meridian/app.json @@ -120,6 +120,18 @@ "expo-av", "expo-secure-store", "expo-local-authentication", + [ + "@livekit/react-native-expo-plugin", + { + "android": { + "audioType": "communication" + }, + "ios": { + "enableMultitaskingCameraAccess": false + } + } + ], + "@config-plugins/react-native-webrtc", [ "expo-build-properties", { diff --git a/apps/meridian/app/(main)/converse.tsx b/apps/meridian/app/(main)/converse.tsx index 12ca61a..ec47572 100644 --- a/apps/meridian/app/(main)/converse.tsx +++ b/apps/meridian/app/(main)/converse.tsx @@ -48,8 +48,10 @@ import type { StationGeo } from '../../lib/stationGeo'; import { fetchRate } from '../../lib/currency'; import { formatMoneyAmount, isZeroDecimalCurrency } from '../../lib/money'; import type { TripContext } from '../../lib/trip'; -import { rememberConfirmApproved, trackClientEvent } from '../../lib/telemetry'; +import { getLaunchSessionId, rememberConfirmApproved, trackClientEvent } from '../../lib/telemetry'; import { loadPreferredTravelUnit, type TravelUnit } from '../../lib/travelUnits'; +import { DEFAULT_VOICE_SESSION_CONFIG, isLiveVoiceSession, loadVoiceSessionConfig } from '../../lib/voiceSession'; +import { connectLiveVoiceSession, type LiveVoiceConnection } from '../../lib/liveVoice'; type MarketNationality = 'uk' | 'india' | 'other'; @@ -765,9 +767,16 @@ export default function ConverseScreen() { const [nearestStation, setNearestStation] = useState(null); const [locationLabel, setLocationLabel] = useState(null); const [voiceEnabled, setVoiceEnabled] = useState(true); + const [voiceSessionConfig, setVoiceSessionConfig] = useState(DEFAULT_VOICE_SESSION_CONFIG); const [isSpeaking, setIsSpeaking] = useState(false); + const [liveVoiceConnected, setLiveVoiceConnected] = useState(false); + const [liveVoiceConnecting, setLiveVoiceConnecting] = useState(false); + const [liveVoiceMicEnabled, setLiveVoiceMicEnabled] = useState(false); + const [liveRemoteSpeaking, setLiveRemoteSpeaking] = useState(false); const micAmplitude = useSharedValue(0); const ttsAmplitude = useSharedValue(0); + const liveVoiceConnectionRef = useRef(null); + const liveVoiceConnectAttemptedRef = useRef(false); const beginVoiceCaptureRef = useRef<(() => Promise) | null>(null); const handsFreeListenTimerRef = useRef | null>(null); const nextSilenceMsRef = useRef(2800); @@ -802,6 +811,96 @@ export default function ConverseScreen() { }); }, []); + const disconnectLiveVoice = useCallback(async () => { + const current = liveVoiceConnectionRef.current; + liveVoiceConnectionRef.current = null; + setLiveVoiceConnected(false); + setLiveVoiceConnecting(false); + setLiveVoiceMicEnabled(false); + setLiveRemoteSpeaking(false); + if (current) { + await current.disconnect().catch(() => {}); + } + }, []); + + const connectLiveVoice = useCallback(async () => { + if (!agentId || !isLiveVoiceSession(voiceSessionConfig)) return; + if (liveVoiceConnectionRef.current || liveVoiceConnecting) return; + + setLiveVoiceConnecting(true); + try { + const connection = await connectLiveVoiceSession({ + hirerId: agentId, + sessionId: getLaunchSessionId(), + callbacks: { + onConnected: () => { + setLiveVoiceConnected(true); + setLiveVoiceConnecting(false); + setLiveVoiceMicEnabled(true); + phaseRef.current = 'listening'; + setPhase('listening'); + logConverseEvent({ + event: 'live_voice_connected', + metadata: { + provider: voiceSessionConfig.provider, + transport: voiceSessionConfig.transport, + }, + }); + }, + onDisconnected: () => { + liveVoiceConnectionRef.current = null; + setLiveVoiceConnected(false); + setLiveVoiceMicEnabled(false); + setLiveRemoteSpeaking(false); + if (phaseRef.current === 'listening') { + phaseRef.current = 'idle'; + setPhase('idle'); + } + logConverseEvent({ event: 'live_voice_disconnected' }); + }, + onReconnecting: () => { + setLiveVoiceConnecting(true); + logConverseEvent({ event: 'live_voice_reconnecting' }); + }, + onReconnected: () => { + setLiveVoiceConnected(true); + setLiveVoiceConnecting(false); + logConverseEvent({ event: 'live_voice_reconnected' }); + }, + onRemoteSpeakingChanged: (speaking) => { + setLiveRemoteSpeaking(speaking); + }, + onError: (error) => { + liveVoiceConnectionRef.current = null; + const message = error.message || 'Live voice failed.'; + setLiveVoiceConnected(false); + setLiveVoiceConnecting(false); + setLiveVoiceMicEnabled(false); + setLiveRemoteSpeaking(false); + logConverseEvent({ + event: 'live_voice_error', + severity: 'warning', + message, + }); + }, + }, + }); + + liveVoiceConnectionRef.current = connection; + } catch (error: any) { + liveVoiceConnectionRef.current = null; + setLiveVoiceConnecting(false); + setLiveVoiceConnected(false); + setLiveVoiceMicEnabled(false); + setLiveRemoteSpeaking(false); + logConverseEvent({ + event: 'live_voice_connect_failed', + severity: 'warning', + message: error?.message ?? 'Live voice failed to connect.', + }); + } + }, [agentId, liveVoiceConnecting, logConverseEvent, setPhase, voiceSessionConfig]); + useEffect(() => { if (turns.length === 0) return; setTimeout(() => scrollRef.current?.scrollToEnd({ animated: true }), 100); @@ -1261,12 +1360,59 @@ export default function ConverseScreen() { // Pre-fetch the opening greeting the moment the screen mounts so the audio // file is ready before the 1.2s timer fires — instant first sound. useEffect(() => { - if (!voiceEnabled) return; + let cancelled = false; + + void loadVoiceSessionConfig() + .then((config) => { + if (cancelled) return; + setVoiceSessionConfig(config); + void trackClientEvent({ + event: 'voice_runtime_configured', + screen: 'converse', + metadata: { + mode: config.mode, + transport: config.transport, + provider: config.provider, + ready: config.ready, + supportsInterruptions: config.supportsInterruptions, + supportsServerVad: config.supportsServerVad, + premiumVoice: config.premiumVoice, + planningToolsAvailableDuringConversation: config.planningToolsAvailableDuringConversation, + bookingToolsLockedUntilConfirm: config.bookingToolsLockedUntilConfirm, + diagnostics: config.diagnostics, + }, + }); + }) + .catch(() => null); + + return () => { + cancelled = true; + }; + }, []); + + useEffect(() => { + if (!agentId || !voiceEnabled || !isLiveVoiceSession(voiceSessionConfig)) return; + if (liveVoiceConnectAttemptedRef.current) return; + liveVoiceConnectAttemptedRef.current = true; + void connectLiveVoice(); + }, [agentId, connectLiveVoice, voiceEnabled, voiceSessionConfig]); + + useEffect(() => { + if (voiceEnabled) return; + void disconnectLiveVoice(); + }, [disconnectLiveVoice, voiceEnabled]); + + useEffect(() => () => { + void disconnectLiveVoice(); + }, [disconnectLiveVoice]); + + useEffect(() => { + if (!voiceEnabled || isLiveVoiceSession(voiceSessionConfig)) return; const h = new Date().getHours(); const line = h < 12 ? 'Good morning.' : h < 17 ? 'Good afternoon.' : 'Good evening.'; void preloadAudio(line); // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); + }, [voiceEnabled, voiceSessionConfig]); // Always-on: auto-start listening 1.2s after mount once agentId is ready. // Skipped if a prefill is pending (it will take priority) or a trip is active. @@ -1280,6 +1426,10 @@ export default function ConverseScreen() { if (liveSession && shouldPreferJourney(liveSession)) return; // Only start if we're still idle and the keyboard isn't up if (phaseRef.current === 'idle' && !keyboardVisibleRef.current && !textFallbackVisibleRef.current) { + if (isLiveVoiceSession(voiceSessionConfig)) { + void connectLiveVoice(); + return; + } // Speak a brief greeting so the user knows Ace is ready, then // speakIfEnabled's restartListening path automatically arms the mic. const h = new Date().getHours(); @@ -1289,10 +1439,10 @@ export default function ConverseScreen() { })(); }, 1200); return () => clearTimeout(timer); - }, [activeTrip, agentId, prefill, voiceEnabled]); + }, [activeTrip, agentId, connectLiveVoice, prefill, voiceEnabled, voiceSessionConfig]); useEffect(() => { - if (phase !== 'done' || isSpeaking) return; + if (phase !== 'done' || isSpeaking || liveRemoteSpeaking) return; const timer = setTimeout(() => { if (phaseRef.current !== 'done') return; setPhase('idle'); @@ -1307,7 +1457,7 @@ export default function ConverseScreen() { } }, 4000); return () => clearTimeout(timer); - }, [activeTrip, isSpeaking, phase, setPhase, voiceEnabled]); + }, [activeTrip, isSpeaking, liveRemoteSpeaking, phase, setPhase, voiceEnabled]); const handleTextFallbackSend = useCallback(async () => { const text = textFallbackDraft.trim(); @@ -1574,6 +1724,19 @@ export default function ConverseScreen() { const finishingRecordingRef = useRef(false); const finishVoiceCapture = useCallback(async () => { + if (isLiveVoiceSession(voiceSessionConfig)) { + clearHandsFreeListenTimer(); + if (liveVoiceConnectionRef.current) { + await liveVoiceConnectionRef.current.setMicrophoneEnabled(false).catch(() => {}); + } + setLiveVoiceMicEnabled(false); + setLiveRemoteSpeaking(false); + micAmplitude.value = 0; + phaseRef.current = 'idle'; + setPhase('idle'); + return; + } + if (finishingRecordingRef.current) return; clearHandsFreeListenTimer(); if (startingRecordingRef.current && !recordingActiveRef.current) { @@ -1725,10 +1888,43 @@ export default function ConverseScreen() { startingRecordingRef.current = false; finishingRecordingRef.current = false; } - }, [clearHandsFreeListenTimer, logConverseEvent, openTextFallback, runIntentWithUiFallback, setError, setPhase]); + }, [clearHandsFreeListenTimer, logConverseEvent, openTextFallback, runIntentWithUiFallback, setError, setPhase, voiceSessionConfig]); const beginVoiceCapture = useCallback(async () => { if (!voiceEnabled) return; + if (isLiveVoiceSession(voiceSessionConfig)) { + const currentPhase = phaseRef.current; + clearHandsFreeListenTimer(); + cancelSpeech(); + if (currentPhase === 'error') reset(); + setTextFallbackVisible(false); + setError(null); + voiceCaptureStartedAtRef.current = Date.now(); + logConverseEvent({ + event: 'voice_capture_started', + metadata: { fromPhase: currentPhase, live: true }, + }); + logConverseEvent({ + event: 'voice_session_start', + metadata: { + fromPhase: currentPhase, + voiceMode: voiceSessionConfig.mode, + voiceTransport: voiceSessionConfig.transport, + voiceProvider: voiceSessionConfig.provider, + }, + }); + + if (!liveVoiceConnectionRef.current) { + await connectLiveVoice(); + } else { + await liveVoiceConnectionRef.current.setMicrophoneEnabled(true).catch(() => {}); + setLiveVoiceMicEnabled(true); + phaseRef.current = 'listening'; + setPhase('listening'); + } + return; + } + if (recordingActiveRef.current || startingRecordingRef.current || finishingRecordingRef.current) return; const currentPhase = phaseRef.current; if (currentPhase !== 'idle' && currentPhase !== 'error') return; @@ -1746,7 +1942,12 @@ export default function ConverseScreen() { }); logConverseEvent({ event: 'voice_session_start', - metadata: { fromPhase: currentPhase }, + metadata: { + fromPhase: currentPhase, + voiceMode: voiceSessionConfig.mode, + voiceTransport: voiceSessionConfig.transport, + voiceProvider: voiceSessionConfig.provider, + }, }); startingRecordingRef.current = true; try { @@ -1784,7 +1985,7 @@ export default function ConverseScreen() { message, }); } - }, [clearHandsFreeListenTimer, finishVoiceCapture, logConverseEvent, reset, setError, setPhase, voiceEnabled]); + }, [clearHandsFreeListenTimer, connectLiveVoice, finishVoiceCapture, logConverseEvent, reset, setError, setPhase, voiceEnabled, voiceSessionConfig]); // Wire beginVoiceCapture into ref so speakIfEnabled can auto-restart without circular dep beginVoiceCaptureRef.current = beginVoiceCapture; @@ -1795,6 +1996,19 @@ export default function ConverseScreen() { const handleOrbTap = useCallback(async () => { clearHandsFreeListenTimer(); + if (isLiveVoiceSession(voiceSessionConfig)) { + if (liveVoiceConnecting) return; + if (!liveVoiceConnectionRef.current) { + await beginVoiceCapture(); + return; + } + if (liveVoiceMicEnabled) { + await finishVoiceCapture(); + return; + } + await beginVoiceCapture(); + return; + } if (phase === 'listening') { await finishVoiceCapture(); return; @@ -1809,7 +2023,7 @@ export default function ConverseScreen() { } nextSilenceMsRef.current = 2800; await beginVoiceCapture(); - }, [beginVoiceCapture, clearHandsFreeListenTimer, finishVoiceCapture, phase]); + }, [beginVoiceCapture, clearHandsFreeListenTimer, finishVoiceCapture, liveVoiceConnecting, liveVoiceMicEnabled, phase, voiceSessionConfig]); const handleShortcutIntent = useCallback(async (destination: string, kind: 'home' | 'work') => { if (!nearestStation) return; @@ -1853,22 +2067,25 @@ export default function ConverseScreen() { const isIdle = phase === 'idle'; const isError = phase === 'error'; const isConfirming = phase === 'confirming'; + const liveVoiceReady = isLiveVoiceSession(voiceSessionConfig); const presenceLabel = - isSpeaking ? 'Ace is speaking' : - phase === 'listening' ? 'Ace is listening' : + isSpeaking || liveRemoteSpeaking ? 'Ace is speaking' : + liveVoiceConnecting ? 'Ace is joining' : + phase === 'listening' ? (liveVoiceReady ? 'Ace is with you' : 'Ace is listening') : phase === 'thinking' ? 'Ace is thinking' : phase === 'hiring' || phase === 'executing' ? 'Ace is securing your trip' : phase === 'done' ? 'Trip secured' : phase === 'error' ? 'Ace is waiting' : - voiceEnabled ? 'Ace is ready' : + voiceEnabled ? (liveVoiceReady ? 'Ace is live' : 'Ace is ready') : 'Voice paused'; const presenceHint = - isSpeaking ? 'Stay with me. Ace is guiding the next move.' : - phase === 'listening' ? 'Say the trip once, naturally.' : + isSpeaking || liveRemoteSpeaking ? 'Stay with me. Ace is guiding the next move.' : + liveVoiceConnecting ? 'Joining your live concierge session.' : + phase === 'listening' ? (liveVoiceReady ? 'Speak naturally. Ace will stay with the thread.' : 'Say the trip once, naturally.') : phase === 'thinking' || phase === 'hiring' || phase === 'executing' ? 'Ace is working the route, timing, and booking.' : phase === 'done' ? 'Trip secured. Ace will stay with it.' : phase === 'error' ? 'Say it again or continue in text below.' : - voiceEnabled ? 'Say the trip when you are ready.' : + voiceEnabled ? (liveVoiceReady ? 'Ace is live. Say the trip when you are ready.' : 'Say the trip when you are ready.') : 'Tap Ace to resume voice.'; const presenceTone = phase === 'listening' ? '#c8e8ff' : diff --git a/apps/meridian/app/_layout.tsx b/apps/meridian/app/_layout.tsx index ffe155c..db4167f 100644 --- a/apps/meridian/app/_layout.tsx +++ b/apps/meridian/app/_layout.tsx @@ -3,6 +3,7 @@ import { Stack, useRouter } from 'expo-router'; import { GestureHandlerRootView } from 'react-native-gesture-handler'; import { StatusBar } from 'expo-status-bar'; import * as Notifications from 'expo-notifications'; +import '../lib/livekitGlobals'; import { trackClientEvent } from '../lib/telemetry'; export default function RootLayout() { diff --git a/apps/meridian/lib/api.ts b/apps/meridian/lib/api.ts index 0ff3253..dd2cf83 100644 --- a/apps/meridian/lib/api.ts +++ b/apps/meridian/lib/api.ts @@ -310,10 +310,53 @@ export interface ConciergeExecutionSnapshot { updatedAt: string | null; } +export interface VoiceSessionConfig { + mode: 'batch' | 'live'; + transport: 'http' | 'webrtc'; + provider: 'batch_proxy' | 'livekit' | 'openai_realtime'; + ready: boolean; + planningToolsAvailableDuringConversation: boolean; + bookingToolsLockedUntilConfirm: boolean; + supportsInterruptions: boolean; + supportsServerVad: boolean; + premiumVoice: 'elevenlabs' | 'system' | 'none'; + fallback: { + stt: 'whisper_proxy'; + tts: 'elevenlabs_http' | 'none'; + }; + diagnostics: string[]; +} + +export interface LiveVoiceSessionCredentials { + mode: 'live'; + provider: 'livekit'; + serverUrl: string; + roomName: string; + participantIdentity: string; + participantToken: string; + expiresAt: string; + planningToolsAvailableDuringConversation: true; + bookingToolsLockedUntilConfirm: true; +} + export async function getConciergeExecution(jobId: string): Promise { return apiFetch(`/api/concierge/executions/${jobId}`); } +export async function getVoiceSessionConfig(): Promise { + return apiFetch('/api/voice/session'); +} + +export async function createLiveVoiceSession(params: { + hirerId: string; + sessionId: string; +}): Promise { + return apiFetch('/api/voice/session/connect', { + method: 'POST', + body: JSON.stringify(params), + }); +} + /** Discover agents (text search) */ export async function discoverAgents(params: { q?: string; diff --git a/apps/meridian/lib/liveVoice.ts b/apps/meridian/lib/liveVoice.ts new file mode 100644 index 0000000..36298a5 --- /dev/null +++ b/apps/meridian/lib/liveVoice.ts @@ -0,0 +1,119 @@ +import { AudioSession } from '@livekit/react-native'; +import { ConnectionState, Room, RoomEvent, type Participant, type RoomConnectOptions } from 'livekit-client'; +import { createLiveVoiceSession } from './api'; + +export type LiveVoiceCallbacks = { + onConnected?: () => void; + onDisconnected?: () => void; + onReconnecting?: () => void; + onReconnected?: () => void; + onConnectionStateChanged?: (state: ConnectionState) => void; + onRemoteSpeakingChanged?: (speaking: boolean) => void; + onParticipantConnected?: (participant: Participant) => void; + onParticipantDisconnected?: (participant: Participant) => void; + onError?: (error: Error) => void; +}; + +export type LiveVoiceConnection = { + room: Room; + roomName: string; + participantIdentity: string; + disconnect: () => Promise; + setMicrophoneEnabled: (enabled: boolean) => Promise; +}; + +export async function connectLiveVoiceSession(params: { + hirerId: string; + sessionId: string; + callbacks?: LiveVoiceCallbacks; +}): Promise { + const credentials = await createLiveVoiceSession({ + hirerId: params.hirerId, + sessionId: params.sessionId, + }); + + const room = new Room({ + adaptiveStream: false, + dynacast: false, + stopLocalTrackOnUnpublish: true, + }); + + const callbacks = params.callbacks; + const emitRemoteSpeaking = () => { + const remoteSpeaking = room.activeSpeakers.some((participant) => participant.identity !== room.localParticipant.identity); + callbacks?.onRemoteSpeakingChanged?.(remoteSpeaking); + }; + + room + .on(RoomEvent.Connected, () => { + callbacks?.onConnected?.(); + emitRemoteSpeaking(); + }) + .on(RoomEvent.Disconnected, () => { + callbacks?.onDisconnected?.(); + callbacks?.onRemoteSpeakingChanged?.(false); + }) + .on(RoomEvent.Reconnecting, () => { + callbacks?.onReconnecting?.(); + }) + .on(RoomEvent.Reconnected, () => { + callbacks?.onReconnected?.(); + emitRemoteSpeaking(); + }) + .on(RoomEvent.ConnectionStateChanged, (state) => { + callbacks?.onConnectionStateChanged?.(state); + }) + .on(RoomEvent.ActiveSpeakersChanged, () => { + emitRemoteSpeaking(); + }) + .on(RoomEvent.ParticipantConnected, (participant) => { + callbacks?.onParticipantConnected?.(participant); + }) + .on(RoomEvent.ParticipantDisconnected, (participant) => { + callbacks?.onParticipantDisconnected?.(participant); + emitRemoteSpeaking(); + }); + + try { + await AudioSession.startAudioSession(); + room.prepareConnection(credentials.serverUrl, credentials.participantToken); + const connectOptions: RoomConnectOptions = { + autoSubscribe: true, + }; + await room.connect(credentials.serverUrl, credentials.participantToken, connectOptions); + await room.localParticipant.setMicrophoneEnabled(true, { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }); + } catch (error: any) { + callbacks?.onError?.(error instanceof Error ? error : new Error(String(error))); + try { + await room.disconnect(); + } catch {} + await AudioSession.stopAudioSession().catch(() => {}); + throw error; + } + + return { + room, + roomName: credentials.roomName, + participantIdentity: credentials.participantIdentity, + disconnect: async () => { + callbacks?.onRemoteSpeakingChanged?.(false); + try { + await room.localParticipant.setMicrophoneEnabled(false).catch(() => {}); + await room.disconnect(); + } finally { + await AudioSession.stopAudioSession().catch(() => {}); + } + }, + setMicrophoneEnabled: async (enabled: boolean) => { + await room.localParticipant.setMicrophoneEnabled(enabled, { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }); + }, + }; +} diff --git a/apps/meridian/lib/livekitGlobals.ts b/apps/meridian/lib/livekitGlobals.ts new file mode 100644 index 0000000..786b0b0 --- /dev/null +++ b/apps/meridian/lib/livekitGlobals.ts @@ -0,0 +1,11 @@ +import { registerGlobals } from '@livekit/react-native'; + +let registered = false; + +export function ensureLiveKitGlobals(): void { + if (registered) return; + registerGlobals(); + registered = true; +} + +ensureLiveKitGlobals(); diff --git a/apps/meridian/lib/telemetry.ts b/apps/meridian/lib/telemetry.ts index 671a684..d51fe2d 100644 --- a/apps/meridian/lib/telemetry.ts +++ b/apps/meridian/lib/telemetry.ts @@ -15,6 +15,10 @@ function enrichMetadata(metadata?: Record): Record { + if (!forceRefresh && cachedVoiceSessionConfig) { + return cachedVoiceSessionConfig; + } + + try { + cachedVoiceSessionConfig = await getVoiceSessionConfig(); + return cachedVoiceSessionConfig; + } catch { + cachedVoiceSessionConfig = DEFAULT_VOICE_SESSION_CONFIG; + return cachedVoiceSessionConfig; + } +} + +export function getCachedVoiceSessionConfig(): VoiceSessionConfig { + return cachedVoiceSessionConfig ?? DEFAULT_VOICE_SESSION_CONFIG; +} + +export function isLiveVoiceSession(config: VoiceSessionConfig): boolean { + return config.mode === 'live' && config.ready; +} diff --git a/apps/meridian/package.json b/apps/meridian/package.json index 561c184..fe4a83c 100644 --- a/apps/meridian/package.json +++ b/apps/meridian/package.json @@ -21,7 +21,11 @@ "postinstall": "node scripts/patch-expo-modules.js" }, "dependencies": { + "@config-plugins/react-native-webrtc": "^14.0.0", "@expo/vector-icons": "~14.0.4", + "@livekit/react-native": "^2.10.0", + "@livekit/react-native-expo-plugin": "^1.0.2", + "@livekit/react-native-webrtc": "^144.0.0", "@react-native-async-storage/async-storage": "1.23.1", "@react-three/fiber": "^8.18.0", "@shopify/react-native-skia": "1.5.0", @@ -48,6 +52,7 @@ "expo-speech": "~13.0.0", "expo-status-bar": "~2.0.0", "expo-updates": "~0.27.5", + "livekit-client": "^2.18.1", "react": "18.3.1", "react-dom": "^18.3.1", "react-native": "0.76.9", From 0ecb4afab8373e9f09f1fbf6a1a0ff87b2826bf1 Mon Sep 17 00:00:00 2001 From: Baskaran Balasubramanian Date: Fri, 10 Apr 2026 04:25:05 +0530 Subject: [PATCH 02/10] tighten live voice continuity --- apps/meridian/app/(main)/converse.tsx | 156 +++++++++++++++++++++++++- apps/meridian/lib/liveVoice.ts | 18 ++- 2 files changed, 171 insertions(+), 3 deletions(-) diff --git a/apps/meridian/app/(main)/converse.tsx b/apps/meridian/app/(main)/converse.tsx index ec47572..2c37348 100644 --- a/apps/meridian/app/(main)/converse.tsx +++ b/apps/meridian/app/(main)/converse.tsx @@ -773,10 +773,14 @@ export default function ConverseScreen() { const [liveVoiceConnecting, setLiveVoiceConnecting] = useState(false); const [liveVoiceMicEnabled, setLiveVoiceMicEnabled] = useState(false); const [liveRemoteSpeaking, setLiveRemoteSpeaking] = useState(false); + const [liveCaption, setLiveCaption] = useState(null); + const [liveHeardLine, setLiveHeardLine] = useState(null); const micAmplitude = useSharedValue(0); const ttsAmplitude = useSharedValue(0); const liveVoiceConnectionRef = useRef(null); const liveVoiceConnectAttemptedRef = useRef(false); + const liveTranscriptBufferRef = useRef(''); + const liveRemoteMessageIdsRef = useRef(new Set()); const beginVoiceCaptureRef = useRef<(() => Promise) | null>(null); const handsFreeListenTimerRef = useRef | null>(null); const nextSilenceMsRef = useRef(2800); @@ -814,10 +818,13 @@ export default function ConverseScreen() { const disconnectLiveVoice = useCallback(async () => { const current = liveVoiceConnectionRef.current; liveVoiceConnectionRef.current = null; + liveTranscriptBufferRef.current = ''; setLiveVoiceConnected(false); setLiveVoiceConnecting(false); setLiveVoiceMicEnabled(false); setLiveRemoteSpeaking(false); + setLiveCaption(null); + setLiveHeardLine(null); if (current) { await current.disconnect().catch(() => {}); } @@ -837,6 +844,7 @@ export default function ConverseScreen() { setLiveVoiceConnected(true); setLiveVoiceConnecting(false); setLiveVoiceMicEnabled(true); + setLiveCaption(null); phaseRef.current = 'listening'; setPhase('listening'); logConverseEvent({ @@ -849,9 +857,11 @@ export default function ConverseScreen() { }, onDisconnected: () => { liveVoiceConnectionRef.current = null; + liveTranscriptBufferRef.current = ''; setLiveVoiceConnected(false); setLiveVoiceMicEnabled(false); setLiveRemoteSpeaking(false); + setLiveCaption(null); if (phaseRef.current === 'listening') { phaseRef.current = 'idle'; setPhase('idle'); @@ -870,6 +880,55 @@ export default function ConverseScreen() { onRemoteSpeakingChanged: (speaking) => { setLiveRemoteSpeaking(speaking); }, + onChatMessage: async (message, participant) => { + if (!participant || participant.identity === agentId) return; + if (liveRemoteMessageIdsRef.current.has(message.id)) return; + liveRemoteMessageIdsRef.current.add(message.id); + const narration = sanitizeAceNarration(message.message); + if (!narration) return; + setLiveCaption(narration); + const meridianTurn = { role: 'meridian' as const, text: narration, ts: Date.now() }; + addTurn(meridianTurn); + await appendHistory(meridianTurn).catch(() => {}); + logConverseEvent({ + event: 'live_voice_message_received', + metadata: { + participantIdentity: participant.identity, + textLength: narration.length, + }, + }); + }, + onTranscriptionReceived: (segments, participant) => { + const joined = segments + .map((segment) => segment.text.trim()) + .filter(Boolean) + .join(' ') + .trim(); + if (!joined) return; + + if (participant?.identity === agentId) { + liveTranscriptBufferRef.current = joined; + setLiveHeardLine(joined); + logConverseEvent({ + event: 'live_voice_local_transcript', + metadata: { + final: segments.every((segment) => segment.final), + textLength: joined.length, + }, + }); + return; + } + + setLiveCaption(joined); + logConverseEvent({ + event: 'live_voice_remote_transcript', + metadata: { + final: segments.every((segment) => segment.final), + textLength: joined.length, + participantIdentity: participant?.identity ?? null, + }, + }); + }, onError: (error) => { liveVoiceConnectionRef.current = null; const message = error.message || 'Live voice failed.'; @@ -877,6 +936,7 @@ export default function ConverseScreen() { setLiveVoiceConnecting(false); setLiveVoiceMicEnabled(false); setLiveRemoteSpeaking(false); + setLiveCaption(null); logConverseEvent({ event: 'live_voice_error', severity: 'warning', @@ -899,7 +959,7 @@ export default function ConverseScreen() { message: error?.message ?? 'Live voice failed to connect.', }); } - }, [agentId, liveVoiceConnecting, logConverseEvent, setPhase, voiceSessionConfig]); + }, [addTurn, agentId, liveVoiceConnecting, logConverseEvent, setPhase, voiceSessionConfig]); useEffect(() => { if (turns.length === 0) return; @@ -1729,8 +1789,20 @@ export default function ConverseScreen() { if (liveVoiceConnectionRef.current) { await liveVoiceConnectionRef.current.setMicrophoneEnabled(false).catch(() => {}); } + if (liveTranscriptBufferRef.current.trim()) { + const heard = liveTranscriptBufferRef.current.trim(); + setTranscript(heard); + addTurn({ role: 'user', text: heard, ts: Date.now() }); + await appendHistory({ role: 'user', text: heard, ts: Date.now() }).catch(() => {}); + logConverseEvent({ + event: 'live_voice_local_turn_captured', + metadata: { textLength: heard.length }, + }); + } + liveTranscriptBufferRef.current = ''; setLiveVoiceMicEnabled(false); setLiveRemoteSpeaking(false); + setLiveHeardLine(null); micAmplitude.value = 0; phaseRef.current = 'idle'; setPhase('idle'); @@ -1888,7 +1960,7 @@ export default function ConverseScreen() { startingRecordingRef.current = false; finishingRecordingRef.current = false; } - }, [clearHandsFreeListenTimer, logConverseEvent, openTextFallback, runIntentWithUiFallback, setError, setPhase, voiceSessionConfig]); + }, [addTurn, clearHandsFreeListenTimer, logConverseEvent, openTextFallback, runIntentWithUiFallback, setError, setPhase, setTranscript, voiceSessionConfig]); const beginVoiceCapture = useCallback(async () => { if (!voiceEnabled) return; @@ -1899,6 +1971,8 @@ export default function ConverseScreen() { if (currentPhase === 'error') reset(); setTextFallbackVisible(false); setError(null); + setLiveHeardLine(null); + liveTranscriptBufferRef.current = ''; voiceCaptureStartedAtRef.current = Date.now(); logConverseEvent({ event: 'voice_capture_started', @@ -2443,6 +2517,39 @@ export default function ConverseScreen() { )} + {liveVoiceReady && (liveVoiceConnected || liveVoiceConnecting || liveCaption || liveHeardLine) && ( + + + Live with Ace + + {liveVoiceConnecting + ? 'Joining' + : liveRemoteSpeaking + ? 'Speaking' + : liveVoiceMicEnabled + ? 'Listening' + : liveVoiceConnected + ? 'Standing by' + : 'Fallback'} + + + {liveHeardLine ? ( + + You: {liveHeardLine} + + ) : null} + {liveCaption ? ( + + Ace: {liveCaption} + + ) : ( + + Ace stays in the room. Planning stays open. Booking opens only when you confirm. + + )} + + )} + {recentTurns.length > 0 && ( @@ -3008,6 +3115,51 @@ const styles = StyleSheet.create({ lineHeight: 19, color: '#91a4b8', }, + liveSessionCard: { + marginBottom: 14, + paddingHorizontal: 16, + paddingVertical: 14, + borderRadius: 22, + borderWidth: 1, + borderColor: 'rgba(118, 190, 236, 0.22)', + backgroundColor: 'rgba(7, 17, 29, 0.84)', + }, + liveSessionHeader: { + flexDirection: 'row', + alignItems: 'center', + justifyContent: 'space-between', + marginBottom: 8, + }, + liveSessionEyebrow: { + fontSize: 11, + color: '#d9ecff', + textTransform: 'uppercase', + letterSpacing: 1.1, + fontWeight: '700', + }, + liveSessionState: { + fontSize: 11, + color: '#8ec4e8', + fontWeight: '700', + letterSpacing: 0.5, + }, + liveSessionHeard: { + fontSize: 13, + lineHeight: 18, + color: '#bfd0df', + marginBottom: 6, + }, + liveSessionCaption: { + fontSize: 14, + lineHeight: 20, + color: '#eef6ff', + fontWeight: '500', + }, + liveSessionHint: { + fontSize: 12, + lineHeight: 18, + color: '#8ea1b5', + }, modeCard: { borderRadius: 22, borderWidth: 1, diff --git a/apps/meridian/lib/liveVoice.ts b/apps/meridian/lib/liveVoice.ts index 36298a5..d8fcc13 100644 --- a/apps/meridian/lib/liveVoice.ts +++ b/apps/meridian/lib/liveVoice.ts @@ -1,5 +1,13 @@ import { AudioSession } from '@livekit/react-native'; -import { ConnectionState, Room, RoomEvent, type Participant, type RoomConnectOptions } from 'livekit-client'; +import { + ConnectionState, + Room, + RoomEvent, + type ChatMessage, + type Participant, + type RoomConnectOptions, + type TranscriptionSegment, +} from 'livekit-client'; import { createLiveVoiceSession } from './api'; export type LiveVoiceCallbacks = { @@ -11,6 +19,8 @@ export type LiveVoiceCallbacks = { onRemoteSpeakingChanged?: (speaking: boolean) => void; onParticipantConnected?: (participant: Participant) => void; onParticipantDisconnected?: (participant: Participant) => void; + onChatMessage?: (message: ChatMessage, participant?: Participant) => void; + onTranscriptionReceived?: (segments: TranscriptionSegment[], participant?: Participant) => void; onError?: (error: Error) => void; }; @@ -72,6 +82,12 @@ export async function connectLiveVoiceSession(params: { .on(RoomEvent.ParticipantDisconnected, (participant) => { callbacks?.onParticipantDisconnected?.(participant); emitRemoteSpeaking(); + }) + .on(RoomEvent.ChatMessage, (message, participant) => { + callbacks?.onChatMessage?.(message, participant); + }) + .on(RoomEvent.TranscriptionReceived, (segments, participant) => { + callbacks?.onTranscriptionReceived?.(segments, participant); }); try { From 65efe9b0bafac7e31847045bea1c65dea93e880e Mon Sep 17 00:00:00 2001 From: Baskaran Balasubramanian Date: Fri, 10 Apr 2026 08:28:24 +0530 Subject: [PATCH 03/10] fix ace voice runtime fallback --- apps/api-edge/src/routes/voice.ts | 25 ++++-- apps/api-edge/src/types.ts | 2 + apps/meridian/app.json | 2 + apps/meridian/app/(main)/converse.tsx | 116 +++++++++++++++++++++----- apps/meridian/lib/liveVoice.ts | 40 +++++++++ 5 files changed, 159 insertions(+), 26 deletions(-) diff --git a/apps/api-edge/src/routes/voice.ts b/apps/api-edge/src/routes/voice.ts index 6c0fa73..f41b627 100644 --- a/apps/api-edge/src/routes/voice.ts +++ b/apps/api-edge/src/routes/voice.ts @@ -33,6 +33,9 @@ type VoiceSessionConnectResponse = { }; function getLiveVoiceProvider(env: Env): VoiceSessionProvider | null { + if (env.LIVE_VOICE_RUNTIME_ENABLED !== 'true') { + return null; + } if (env.LIVEKIT_URL && env.LIVEKIT_API_KEY && env.LIVEKIT_API_SECRET) { return 'livekit'; } @@ -117,14 +120,18 @@ function getVoiceSessionManifest(env: Env): { const liveProvider = getLiveVoiceProvider(env); const diagnostics: string[] = []; - if (!liveProvider) { + if (env.LIVE_VOICE_RUNTIME_ENABLED !== 'true') { + diagnostics.push('live_runtime_disabled'); + } else if (!liveProvider) { diagnostics.push('live_runtime_not_configured'); - if (!env.LIVEKIT_URL || !env.LIVEKIT_API_KEY || !env.LIVEKIT_API_SECRET) { - diagnostics.push('livekit_credentials_missing'); - } - if (!env.OPENAI_REALTIME_MODEL) { - diagnostics.push('openai_realtime_model_missing'); - } + } + + if (!env.LIVEKIT_URL || !env.LIVEKIT_API_KEY || !env.LIVEKIT_API_SECRET) { + diagnostics.push('livekit_credentials_missing'); + } + + if (!env.OPENAI_REALTIME_MODEL) { + diagnostics.push('openai_realtime_model_missing'); } if (!env.ELEVENLABS_API_KEY) { @@ -230,6 +237,10 @@ voiceRouter.post('/session/connect', async (c) => { const unauthorized = requireBroClientKey(c); if (unauthorized) return unauthorized; + if (c.env.LIVE_VOICE_RUNTIME_ENABLED !== 'true') { + return c.json({ error: 'Live voice runtime is disabled' }, 503); + } + const { LIVEKIT_URL, LIVEKIT_API_KEY, LIVEKIT_API_SECRET } = c.env; if (!LIVEKIT_URL || !LIVEKIT_API_KEY || !LIVEKIT_API_SECRET) { return c.json({ error: 'Live voice session is not configured' }, 503); diff --git a/apps/api-edge/src/types.ts b/apps/api-edge/src/types.ts index 11bcd77..33614e0 100644 --- a/apps/api-edge/src/types.ts +++ b/apps/api-edge/src/types.ts @@ -220,6 +220,8 @@ export interface Env { LIVEKIT_API_KEY?: string; /** LiveKit API secret paired with LIVEKIT_API_KEY. */ LIVEKIT_API_SECRET?: string; + /** Explicit gate for the live voice runtime. Keep false until an Ace participant actually joins rooms and publishes audio. */ + LIVE_VOICE_RUNTIME_ENABLED?: string; /** Optional OpenAI Realtime model name for live speech sessions. */ OPENAI_REALTIME_MODEL?: string; /** Optional ElevenLabs voice/agent identifier for live voice rendering. */ diff --git a/apps/meridian/app.json b/apps/meridian/app.json index dc44bd9..8903c2e 100644 --- a/apps/meridian/app.json +++ b/apps/meridian/app.json @@ -102,6 +102,8 @@ "android.permission.USE_FINGERPRINT", "android.permission.USE_BIOMETRIC", "android.permission.RECORD_AUDIO", + "android.permission.BLUETOOTH", + "android.permission.BLUETOOTH_CONNECT", "android.permission.RECEIVE_BOOT_COMPLETED", "android.permission.VIBRATE", "android.permission.ACCESS_COARSE_LOCATION", diff --git a/apps/meridian/app/(main)/converse.tsx b/apps/meridian/app/(main)/converse.tsx index 2c37348..c7a67fd 100644 --- a/apps/meridian/app/(main)/converse.tsx +++ b/apps/meridian/app/(main)/converse.tsx @@ -780,8 +780,11 @@ export default function ConverseScreen() { const liveVoiceConnectionRef = useRef(null); const liveVoiceConnectAttemptedRef = useRef(false); const liveTranscriptBufferRef = useRef(''); + const liveRemotePresenceSeenRef = useRef(false); const liveRemoteMessageIdsRef = useRef(new Set()); + const liveVoicePresenceTimerRef = useRef | null>(null); const beginVoiceCaptureRef = useRef<(() => Promise) | null>(null); + const speakIfEnabledRef = useRef<((text: string, restartListening?: boolean) => Promise) | null>(null); const handsFreeListenTimerRef = useRef | null>(null); const nextSilenceMsRef = useRef(2800); const keyboardVisibleRef = useRef(false); @@ -815,10 +818,65 @@ export default function ConverseScreen() { }); }, []); + const clearLiveVoicePresenceTimer = useCallback(() => { + if (liveVoicePresenceTimerRef.current) { + clearTimeout(liveVoicePresenceTimerRef.current); + liveVoicePresenceTimerRef.current = null; + } + }, []); + + const surfaceLiveVoiceFallback = useCallback((message?: string) => { + const nextMessage = message?.trim() || 'Live voice is unavailable right now. Type the trip below and Ace will keep moving.'; + clearLiveVoicePresenceTimer(); + liveVoiceConnectionRef.current = null; + liveTranscriptBufferRef.current = ''; + liveRemotePresenceSeenRef.current = false; + setLiveVoiceConnected(false); + setLiveVoiceConnecting(false); + setLiveVoiceMicEnabled(false); + setLiveRemoteSpeaking(false); + setLiveCaption(null); + setLiveHeardLine(null); + setVoiceSessionConfig(DEFAULT_VOICE_SESSION_CONFIG); + phaseRef.current = 'error'; + setPhase('error'); + setError(nextMessage); + setTextFallbackVisible(true); + }, [clearLiveVoicePresenceTimer, setError, setPhase]); + + const degradeLiveVoiceToBatch = useCallback(async () => { + clearLiveVoicePresenceTimer(); + const current = liveVoiceConnectionRef.current; + liveVoiceConnectionRef.current = null; + liveTranscriptBufferRef.current = ''; + liveRemotePresenceSeenRef.current = false; + setLiveVoiceConnected(false); + setLiveVoiceConnecting(false); + setLiveVoiceMicEnabled(false); + setLiveRemoteSpeaking(false); + setLiveCaption(null); + setLiveHeardLine(null); + setVoiceSessionConfig(DEFAULT_VOICE_SESSION_CONFIG); + setError(null); + setTextFallbackVisible(false); + phaseRef.current = 'idle'; + setPhase('idle'); + + if (current) { + await current.disconnect().catch(() => {}); + } + + const hour = new Date().getHours(); + const openingLine = hour < 12 ? 'Good morning.' : hour < 17 ? 'Good afternoon.' : 'Good evening.'; + await speakIfEnabledRef.current?.(openingLine, true); + }, [clearLiveVoicePresenceTimer, setError, setPhase]); + const disconnectLiveVoice = useCallback(async () => { const current = liveVoiceConnectionRef.current; + clearLiveVoicePresenceTimer(); liveVoiceConnectionRef.current = null; liveTranscriptBufferRef.current = ''; + liveRemotePresenceSeenRef.current = false; setLiveVoiceConnected(false); setLiveVoiceConnecting(false); setLiveVoiceMicEnabled(false); @@ -828,12 +886,14 @@ export default function ConverseScreen() { if (current) { await current.disconnect().catch(() => {}); } - }, []); + }, [clearLiveVoicePresenceTimer]); const connectLiveVoice = useCallback(async () => { if (!agentId || !isLiveVoiceSession(voiceSessionConfig)) return; if (liveVoiceConnectionRef.current || liveVoiceConnecting) return; + clearLiveVoicePresenceTimer(); + liveRemotePresenceSeenRef.current = false; setLiveVoiceConnecting(true); try { const connection = await connectLiveVoiceSession({ @@ -854,10 +914,22 @@ export default function ConverseScreen() { transport: voiceSessionConfig.transport, }, }); + clearLiveVoicePresenceTimer(); + liveVoicePresenceTimerRef.current = setTimeout(() => { + if (liveRemotePresenceSeenRef.current) return; + logConverseEvent({ + event: 'live_voice_room_empty', + severity: 'warning', + message: 'Live room connected without an Ace participant. Falling back to batch voice.', + }); + void degradeLiveVoiceToBatch(); + }, 2500); }, onDisconnected: () => { liveVoiceConnectionRef.current = null; liveTranscriptBufferRef.current = ''; + clearLiveVoicePresenceTimer(); + liveRemotePresenceSeenRef.current = false; setLiveVoiceConnected(false); setLiveVoiceMicEnabled(false); setLiveRemoteSpeaking(false); @@ -875,15 +947,23 @@ export default function ConverseScreen() { onReconnected: () => { setLiveVoiceConnected(true); setLiveVoiceConnecting(false); + clearLiveVoicePresenceTimer(); logConverseEvent({ event: 'live_voice_reconnected' }); }, onRemoteSpeakingChanged: (speaking) => { + if (speaking) { + liveRemotePresenceSeenRef.current = true; + clearLiveVoicePresenceTimer(); + } setLiveRemoteSpeaking(speaking); }, onChatMessage: async (message, participant) => { - if (!participant || participant.identity === agentId) return; + const localParticipantIdentity = liveVoiceConnectionRef.current?.participantIdentity; + if (!participant || participant.identity === localParticipantIdentity) return; if (liveRemoteMessageIdsRef.current.has(message.id)) return; liveRemoteMessageIdsRef.current.add(message.id); + liveRemotePresenceSeenRef.current = true; + clearLiveVoicePresenceTimer(); const narration = sanitizeAceNarration(message.message); if (!narration) return; setLiveCaption(narration); @@ -899,6 +979,7 @@ export default function ConverseScreen() { }); }, onTranscriptionReceived: (segments, participant) => { + const localParticipantIdentity = liveVoiceConnectionRef.current?.participantIdentity; const joined = segments .map((segment) => segment.text.trim()) .filter(Boolean) @@ -906,7 +987,7 @@ export default function ConverseScreen() { .trim(); if (!joined) return; - if (participant?.identity === agentId) { + if (participant?.identity === localParticipantIdentity) { liveTranscriptBufferRef.current = joined; setLiveHeardLine(joined); logConverseEvent({ @@ -919,6 +1000,8 @@ export default function ConverseScreen() { return; } + liveRemotePresenceSeenRef.current = true; + clearLiveVoicePresenceTimer(); setLiveCaption(joined); logConverseEvent({ event: 'live_voice_remote_transcript', @@ -931,16 +1014,12 @@ export default function ConverseScreen() { }, onError: (error) => { liveVoiceConnectionRef.current = null; - const message = error.message || 'Live voice failed.'; - setLiveVoiceConnected(false); - setLiveVoiceConnecting(false); - setLiveVoiceMicEnabled(false); - setLiveRemoteSpeaking(false); - setLiveCaption(null); + const message = 'Live voice is unavailable right now. Type the trip below and Ace will keep moving.'; + surfaceLiveVoiceFallback(message); logConverseEvent({ event: 'live_voice_error', severity: 'warning', - message, + message: error.message || message, }); }, }, @@ -949,17 +1028,17 @@ export default function ConverseScreen() { liveVoiceConnectionRef.current = connection; } catch (error: any) { liveVoiceConnectionRef.current = null; - setLiveVoiceConnecting(false); - setLiveVoiceConnected(false); - setLiveVoiceMicEnabled(false); - setLiveRemoteSpeaking(false); + clearLiveVoicePresenceTimer(); + liveRemotePresenceSeenRef.current = false; + const message = 'Ace could not open live voice just now. Type the trip below and it will still handle the route.'; + surfaceLiveVoiceFallback(message); logConverseEvent({ event: 'live_voice_connect_failed', severity: 'warning', - message: error?.message ?? 'Live voice failed to connect.', + message: error?.message ?? message, }); } - }, [addTurn, agentId, liveVoiceConnecting, logConverseEvent, setPhase, voiceSessionConfig]); + }, [addTurn, agentId, clearLiveVoicePresenceTimer, degradeLiveVoiceToBatch, liveVoiceConnecting, logConverseEvent, setPhase, surfaceLiveVoiceFallback, voiceSessionConfig]); useEffect(() => { if (turns.length === 0) return; @@ -2065,7 +2144,6 @@ export default function ConverseScreen() { beginVoiceCaptureRef.current = beginVoiceCapture; // Wire speakIfEnabled into ref so the always-on greeting can use the latest closure - const speakIfEnabledRef = useRef<((text: string, restartListening?: boolean) => Promise) | null>(null); speakIfEnabledRef.current = speakIfEnabled; const handleOrbTap = useCallback(async () => { @@ -2802,14 +2880,14 @@ export default function ConverseScreen() { )} - {textFallbackVisible && ( + {(textFallbackVisible || isError) && ( Continue in text - Keep it short. Ace will still handle the rest. + Type the trip once. Ace will still handle the route, timing, and confirmation from here. { + const { granted } = await Audio.requestPermissionsAsync(); + if (!granted) { + throw new Error('Microphone permission denied.'); + } + + if (Platform.OS === 'android') { + try { + await Audio.setAudioModeAsync({ + allowsRecordingIOS: false, + playThroughEarpieceAndroid: false, + staysActiveInBackground: false, + }); + } catch { + // Non-fatal. LiveKit will still attempt to claim the communication route. + } + + await new Promise((resolve) => setTimeout(resolve, 80)); + } +} + +async function releaseLiveVoiceAudio(): Promise { + if (Platform.OS !== 'android') return; + + try { + await Audio.setAudioModeAsync({ + allowsRecordingIOS: false, + playThroughEarpieceAndroid: false, + staysActiveInBackground: false, + }); + } catch { + // Ignore teardown errors. The next voice path will reset the route again. + } +} + export type LiveVoiceCallbacks = { onConnected?: () => void; onDisconnected?: () => void; @@ -91,6 +128,7 @@ export async function connectLiveVoiceSession(params: { }); try { + await prepareLiveVoiceAudio(); await AudioSession.startAudioSession(); room.prepareConnection(credentials.serverUrl, credentials.participantToken); const connectOptions: RoomConnectOptions = { @@ -108,6 +146,7 @@ export async function connectLiveVoiceSession(params: { await room.disconnect(); } catch {} await AudioSession.stopAudioSession().catch(() => {}); + await releaseLiveVoiceAudio(); throw error; } @@ -122,6 +161,7 @@ export async function connectLiveVoiceSession(params: { await room.disconnect(); } finally { await AudioSession.stopAudioSession().catch(() => {}); + await releaseLiveVoiceAudio(); } }, setMicrophoneEnabled: async (enabled: boolean) => { From 12d742b4810a5b8689f05023dc9a60d95176655f Mon Sep 17 00:00:00 2001 From: Baskaran Balasubramanian Date: Fri, 10 Apr 2026 09:34:58 +0530 Subject: [PATCH 04/10] fallback to system voice when premium tts fails --- apps/meridian/lib/tts.ts | 42 ++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/apps/meridian/lib/tts.ts b/apps/meridian/lib/tts.ts index da36da8..8160ea4 100644 --- a/apps/meridian/lib/tts.ts +++ b/apps/meridian/lib/tts.ts @@ -175,6 +175,26 @@ async function requestVoiceAudio(text: string): Promise { return arrayBuffer ? arrayBufferToBase64(arrayBuffer) : null; } +async function playFallbackSystemVoice( + text: string, + startedAt: number, + reason: string, + options?: SpeakBroOptions, +): Promise { + void trackClientEvent({ + event: 'tts_fallback_system', + screen: 'voice', + severity: 'warning', + message: reason, + metadata: { + latencyMs: Date.now() - startedAt, + textLength: text.length, + platform: Platform.OS, + }, + }); + await playSystemVoice(text, options); +} + async function playSystemVoice(text: string, options?: SpeakBroOptions): Promise { await Audio.setAudioModeAsync({ allowsRecordingIOS: false, @@ -238,14 +258,12 @@ export async function speakBro(text: string, options?: SpeakBroOptions): Promise } else { const audioBase64 = await requestVoiceAudio(trimmed); if (!audioBase64) { - // ElevenLabs unavailable — resolve silently. Premium product never - // speaks with a robotic system voice; the face/mic state still updates. - options?.onMeter?.(0); - void trackClientEvent({ - event: 'tts_skipped_unavailable', - screen: 'voice', - metadata: { latencyMs: Date.now() - startedAt, textLength: trimmed.length }, - }); + await playFallbackSystemVoice( + trimmed, + startedAt, + BRO_KEY ? 'elevenlabs_unavailable' : 'bro_key_missing', + options, + ); return; } uri = `${FileSystem.cacheDirectory}ace_voice_${Date.now()}.mp3`; @@ -347,8 +365,6 @@ export async function speakBro(text: string, options?: SpeakBroOptions): Promise } catch (error: any) { options?.onMeter?.(0); await stopActivePlayback(); - // Silent fail — no robotic iOS fallback. Premium product stays silent - // rather than breaking the voice character. void trackClientEvent({ event: 'tts_failed', screen: 'voice', @@ -356,6 +372,12 @@ export async function speakBro(text: string, options?: SpeakBroOptions): Promise message: error?.message ?? 'Server voice playback failed.', metadata: { provider: 'server', textLength: trimmed.length }, }); + await playFallbackSystemVoice( + trimmed, + startedAt, + error?.message ?? 'server_voice_playback_failed', + options, + ); } } From ed14b9e28c1eb5b7b6aa7964a83534dd401d6d17 Mon Sep 17 00:00:00 2001 From: Baskaran Balasubramanian Date: Fri, 10 Apr 2026 09:35:49 +0530 Subject: [PATCH 05/10] keep ace voice premium only --- apps/meridian/lib/tts.ts | 40 ++++++++-------------------------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/apps/meridian/lib/tts.ts b/apps/meridian/lib/tts.ts index 8160ea4..c9b1621 100644 --- a/apps/meridian/lib/tts.ts +++ b/apps/meridian/lib/tts.ts @@ -175,26 +175,6 @@ async function requestVoiceAudio(text: string): Promise { return arrayBuffer ? arrayBufferToBase64(arrayBuffer) : null; } -async function playFallbackSystemVoice( - text: string, - startedAt: number, - reason: string, - options?: SpeakBroOptions, -): Promise { - void trackClientEvent({ - event: 'tts_fallback_system', - screen: 'voice', - severity: 'warning', - message: reason, - metadata: { - latencyMs: Date.now() - startedAt, - textLength: text.length, - platform: Platform.OS, - }, - }); - await playSystemVoice(text, options); -} - async function playSystemVoice(text: string, options?: SpeakBroOptions): Promise { await Audio.setAudioModeAsync({ allowsRecordingIOS: false, @@ -258,12 +238,14 @@ export async function speakBro(text: string, options?: SpeakBroOptions): Promise } else { const audioBase64 = await requestVoiceAudio(trimmed); if (!audioBase64) { - await playFallbackSystemVoice( - trimmed, - startedAt, - BRO_KEY ? 'elevenlabs_unavailable' : 'bro_key_missing', - options, - ); + options?.onMeter?.(0); + void trackClientEvent({ + event: 'tts_skipped_unavailable', + screen: 'voice', + severity: 'warning', + message: BRO_KEY ? 'elevenlabs_unavailable' : 'bro_key_missing', + metadata: { latencyMs: Date.now() - startedAt, textLength: trimmed.length }, + }); return; } uri = `${FileSystem.cacheDirectory}ace_voice_${Date.now()}.mp3`; @@ -372,12 +354,6 @@ export async function speakBro(text: string, options?: SpeakBroOptions): Promise message: error?.message ?? 'Server voice playback failed.', metadata: { provider: 'server', textLength: trimmed.length }, }); - await playFallbackSystemVoice( - trimmed, - startedAt, - error?.message ?? 'server_voice_playback_failed', - options, - ); } } From de8865020202cc264b1e1a7e703af0ec76e622d1 Mon Sep 17 00:00:00 2001 From: Baskaran Balasubramanian Date: Fri, 10 Apr 2026 09:53:26 +0530 Subject: [PATCH 06/10] tighten solo and family booking clarity --- apps/meridian/app/(main)/converse.tsx | 70 ++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 6 deletions(-) diff --git a/apps/meridian/app/(main)/converse.tsx b/apps/meridian/app/(main)/converse.tsx index c7a67fd..0250bb1 100644 --- a/apps/meridian/app/(main)/converse.tsx +++ b/apps/meridian/app/(main)/converse.tsx @@ -495,6 +495,19 @@ function buildSharedTravelReadiness(unit: TravelUnit | null): SharedTravelReadin }; } +function buildPassengerDisplay( + passengers: Array<{ name: string; relationship: string }> | null, + fallback: string, +): string { + if (!passengers || passengers.length === 0) return fallback; + const names = passengers + .map((passenger) => passenger.name?.trim()) + .filter(Boolean); + if (names.length === 0) return fallback; + if (names.length <= 3) return names.join(', '); + return `${names.slice(0, 2).join(', ')} +${names.length - 2} more`; +} + function getCountdown(departureTime: string | null | undefined): string | null { if (!departureTime) return null; const d = new Date(departureTime); @@ -2276,6 +2289,10 @@ export default function ConverseScreen() { const heroResponse = bookingMode === 'shared' ? `I'll line this up for ${preferredTravelUnit?.name?.toLowerCase() ?? 'both of you'}.` + : !preferredTravelUnit && familyMemberCount >= 2 + ? 'I can move just for you now, or set up family mode once so “for the family” becomes natural.' + : !preferredTravelUnit && familyMemberCount === 1 + ? 'I can move just for you now, or set up couple mode once so “for us” becomes natural.' : routeMemory ? `I can line up ${routeMemory.origin} to ${routeMemory.destination} again, or take you somewhere new.` : nearestStation @@ -2291,16 +2308,15 @@ export default function ConverseScreen() { ? `Ace has seen this route ${routeMemory.count} times and can line it up before you ask tomorrow.` : `Ace remembers this route and can line it up quickly when you need it.` : null; - const shouldShowTravelModeReminder = guidanceSessions >= 3 && guidanceSessions % 6 === 0; - const shouldNudgeTravelSetup = - !preferredTravelUnit - && !travelModePromptDismissed - && guidanceSessions >= 2 - && guidanceSessions % 4 === 0; const inferredTravelShape = familyMemberCount >= 2 ? 'family' : familyMemberCount === 1 ? 'couple' : 'single'; + const shouldShowTravelModeReminder = guidanceSessions >= 3 && guidanceSessions % 6 === 0; + const shouldNudgeTravelSetup = + !preferredTravelUnit + && !travelModePromptDismissed + && inferredTravelShape !== 'single'; const travelModeReminder = preferredTravelUnit ? shouldShowTravelModeReminder ? { @@ -2738,6 +2754,12 @@ export default function ConverseScreen() { const companionSummary = travellerCount > 1 ? `${travellerCount} travellers${childCount > 0 ? `, including ${childCount} child${childCount === 1 ? '' : 'ren'}` : ''}` : null; + const passengerDisplay = buildPassengerDisplay(passengers, travellerCount > 1 ? `${travellerCount} travellers` : 'You'); + const bookingScopeLabel = bookingMode === 'shared' + ? `Holding this for ${preferredTravelUnit?.name ?? companionSummary ?? passengerDisplay}.` + : travellerCount > 1 + ? `Holding this just on your device for ${passengerDisplay}.` + : 'Holding this just for you.'; const hotel = plan.length === 1 ? plan[0]?.hotelDetails?.bestOption : undefined; const isMultiLeg = plan.length > 1; const itinerarySummary = isMultiLeg @@ -2765,6 +2787,15 @@ export default function ConverseScreen() { {confirmMeta ? ( {confirmMeta} ) : null} + + 1 ? 'people-outline' : 'person-outline'} size={13} color="#cfe4f6" /> + + + {travellerCount > 1 ? `Travellers: ${passengerDisplay}` : 'Traveller: You'} + + {bookingScopeLabel} + + {brief && brief.cues.length > 0 && ( {brief.cues.map((cue, idx) => ( @@ -3645,6 +3676,33 @@ const styles = StyleSheet.create({ fontWeight: '600', letterSpacing: 0.3, }, + confirmScopeCard: { + flexDirection: 'row', + alignItems: 'flex-start', + gap: 10, + backgroundColor: 'rgba(9, 17, 28, 0.82)', + borderWidth: 1, + borderColor: 'rgba(122, 167, 214, 0.16)', + borderRadius: 14, + paddingHorizontal: 12, + paddingVertical: 11, + marginBottom: 14, + }, + confirmScopeCopy: { + flex: 1, + gap: 3, + }, + confirmScopeTitle: { + fontSize: 12, + color: '#eef5fb', + fontWeight: '700', + letterSpacing: 0.2, + }, + confirmScopeBody: { + fontSize: 12, + color: '#9db1c5', + lineHeight: 17, + }, confirmBtn: { borderRadius: 18, overflow: 'hidden', From 4da97d97d100e5e3d9c93b788252ddc42c2def6f Mon Sep 17 00:00:00 2001 From: Baskaran Balasubramanian Date: Fri, 10 Apr 2026 10:00:40 +0530 Subject: [PATCH 07/10] tighten shared traveller intent and setup --- apps/meridian/app/(main)/converse.tsx | 238 ++++++++++++++++--- apps/meridian/app/(main)/travel-together.tsx | 96 +++++++- 2 files changed, 305 insertions(+), 29 deletions(-) diff --git a/apps/meridian/app/(main)/converse.tsx b/apps/meridian/app/(main)/converse.tsx index 0250bb1..0351674 100644 --- a/apps/meridian/app/(main)/converse.tsx +++ b/apps/meridian/app/(main)/converse.tsx @@ -41,7 +41,7 @@ import { speakBro, cancelSpeech, preloadAudio } from '../../lib/tts'; import { appendHistory, deriveProactiveRouteMemory, loadActiveTrip, loadCurrentJourneySession, loadRouteMemories, saveJourneySession, type ActiveTrip, type RouteMemory } from '../../lib/storage'; import { planIntent, executeIntent, type ConciergePlanItem } from '../../lib/concierge'; import { shouldPreferJourney, shouldTreatTripAsLive } from '../../lib/journeyRouting'; -import { loadProfileRaw, loadProfileAuthenticated, hasProfile, type TravelProfile } from '../../lib/profile'; +import { loadProfileRaw, loadProfileAuthenticated, hasProfile, type FamilyMember, type TravelProfile } from '../../lib/profile'; import { authenticateWithBiometrics } from '../../lib/biometric'; import { getLocationContext } from '../../lib/location'; import type { StationGeo } from '../../lib/stationGeo'; @@ -103,6 +103,17 @@ type FollowUpContext = { plan: ConciergePlanItem[]; }; +type PassengerIntentTraveller = { + name: string; + relationship: 'adult' | 'child' | 'infant'; +}; + +type PassengerIntent = { + mode: BookingMode; + travellers: PassengerIntentTraveller[]; + summaryLine: string | null; +}; + function looksLikeOptionSelection(text: string): boolean { const normalized = text.trim().toLowerCase(); if (!normalized) return false; @@ -425,6 +436,136 @@ function referencesSharedTravel(text: string): boolean { ].some((pattern) => pattern.test(text)); } +function escapeRegExp(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function firstName(name: string): string { + return name.trim().split(/\s+/)[0] ?? name.trim(); +} + +function nameAppearsInText(text: string, name: string): boolean { + const trimmed = name.trim(); + if (!trimmed) return false; + const normalizedText = ` ${text.toLowerCase()} `; + const fullNamePattern = new RegExp(`(^|[^a-z])${escapeRegExp(trimmed.toLowerCase())}([^a-z]|$)`, 'i'); + if (fullNamePattern.test(normalizedText)) return true; + + const givenName = firstName(trimmed); + if (givenName.length < 3 || givenName.toLowerCase() === 'you') return false; + const givenNamePattern = new RegExp(`(^|[^a-z])${escapeRegExp(givenName.toLowerCase())}([^a-z]|$)`, 'i'); + return givenNamePattern.test(normalizedText); +} + +function familyMemberToTraveller(member: FamilyMember): PassengerIntentTraveller { + return { + name: member.name.trim(), + relationship: member.relationship, + }; +} + +function travelUnitMemberToTraveller(member: TravelUnit['members'][number]): PassengerIntentTraveller | null { + if (member.role === 'self') { + return { + name: 'You', + relationship: 'adult', + }; + } + + const name = member.name.trim(); + if (!name) return null; + return { + name, + relationship: member.role === 'child' || member.role === 'infant' ? member.role : 'adult', + }; +} + +function dedupeTravellers(travellers: PassengerIntentTraveller[]): PassengerIntentTraveller[] { + const seen = new Set(); + const unique: PassengerIntentTraveller[] = []; + for (const traveller of travellers) { + const key = `${traveller.name.trim().toLowerCase()}::${traveller.relationship}`; + if (!traveller.name.trim() || seen.has(key)) continue; + seen.add(key); + unique.push(traveller); + } + return unique; +} + +function describeTravellers(travellers: PassengerIntentTraveller[]): string { + if (travellers.length === 0) return 'you'; + const labels = travellers.map((traveller) => traveller.name === 'You' ? 'you' : traveller.name); + if (labels.length === 1) return labels[0]; + if (labels.length === 2) return `${labels[0]} and ${labels[1]}`; + return `${labels.slice(0, -1).join(', ')}, and ${labels.at(-1)}`; +} + +function buildPassengerIntentPlanningNote(intent: PassengerIntent): string { + if (intent.mode !== 'shared' || intent.travellers.length <= 1 || !intent.summaryLine) { + return ` + +Ace traveller context: +- This request is for the device owner only. +- Do not assume saved companions or family members are travelling unless the customer names them or asks for a shared trip.`; + } + return ` + +Ace traveller context: +- ${intent.summaryLine} +- Treat pricing, availability, and booking scope as a ${intent.travellers.length}-traveller request for ${describeTravellers(intent.travellers)}.`; +} + +function inferPassengerIntent(params: { + text: string; + bookingMode: BookingMode; + preferredTravelUnit: TravelUnit | null; + familyMembers: FamilyMember[]; +}): PassengerIntent { + const familyTravellers = params.familyMembers + .filter((member) => member.name.trim()) + .map(familyMemberToTraveller); + const sharedUnitTravellers = params.preferredTravelUnit?.members + .map(travelUnitMemberToTraveller) + .filter((traveller): traveller is PassengerIntentTraveller => !!traveller) ?? []; + const matchedFamilyTravellers = familyTravellers.filter((traveller) => nameAppearsInText(params.text, traveller.name)); + const matchedSharedTravellers = sharedUnitTravellers.filter((traveller) => traveller.name !== 'You' && nameAppearsInText(params.text, traveller.name)); + const namedTravellers = dedupeTravellers([{ name: 'You', relationship: 'adult' }, ...matchedSharedTravellers, ...matchedFamilyTravellers]); + + if (namedTravellers.length > 1) { + return { + mode: 'shared', + travellers: namedTravellers, + summaryLine: `The customer explicitly asked Ace to book for ${describeTravellers(namedTravellers)}.`, + }; + } + + if (params.preferredTravelUnit && (params.bookingMode === 'shared' || referencesSharedTravel(params.text))) { + const travellers = dedupeTravellers(sharedUnitTravellers); + if (travellers.length > 1) { + return { + mode: 'shared', + travellers, + summaryLine: `Use the saved shared travel unit ${params.preferredTravelUnit.name} for this booking.`, + }; + } + } + + if (referencesSharedTravel(params.text) && familyTravellers.length > 0) { + const travellers = dedupeTravellers([{ name: 'You', relationship: 'adult' }, ...familyTravellers]); + return { + mode: 'shared', + travellers, + summaryLine: `The customer asked Ace to book shared travel for ${describeTravellers(travellers)}.`, + }; + } + + return { + mode: 'solo', + travellers: [{ name: 'You', relationship: 'adult' }], + summaryLine: null, + }; +} + function buildSharedTravelUnitContext(unit: TravelUnit): Record { return { id: unit.id, @@ -1313,12 +1454,10 @@ export default function ConverseScreen() { const followUpContext = shouldUseFollowUpContext(displayText, followUpContextRef.current) ? followUpContextRef.current : null; - const planningTranscript = buildFollowUpPlanningTranscript({ + const basePlanningTranscript = buildFollowUpPlanningTranscript({ reply: prepared.planningTranscript, context: followUpContext, }); - const usingSharedTravel = - !!preferredTravelUnit && (bookingMode === 'shared' || referencesSharedTravel(displayText)); setTextFallbackVisible(false); setTextFallbackDraft(''); @@ -1326,25 +1465,27 @@ export default function ConverseScreen() { setTranscript(displayText); setPhase('thinking'); const planStartedAt = Date.now(); - logConverseEvent({ - event: 'plan_requested', - metadata: { - bookingMode: usingSharedTravel ? 'shared' : 'solo', - textLength: displayText.length, - usedFollowUpContext: !!followUpContext, - usedLocationAssumption: !!prepared.assumptionNote, - }, - }); // Load travel profile for Phase 1 — preferences only, no identity data. // Full profile (legalName, email, phone, documents) is loaded AFTER biometric // confirmation in handleBiometricConfirm. This ensures sensitive data never // reaches memory before the user explicitly authorises the booking. let profile: TravelProfile | null = null; + let passengerIntent: PassengerIntent = { + mode: 'solo', + travellers: [{ name: 'You', relationship: 'adult' }], + summaryLine: null, + }; let travelProfile: Record | undefined; try { if (await hasProfile()) { profile = await loadProfileRaw(); + passengerIntent = inferPassengerIntent({ + text: displayText, + bookingMode, + preferredTravelUnit, + familyMembers: profile?.familyMembers ?? [], + }); if (profile) { // Phase 1: non-identity prefs only — railcard/class/nationality for narration personalisation. // familyMembers included (no documents or contact details) so Claude can resolve names. @@ -1363,8 +1504,10 @@ export default function ConverseScreen() { railcard: m.railcard, nationality: m.nationality, })), + requestedTravellers: passengerIntent.travellers, + requestedTravelMode: passengerIntent.mode, }; - if (usingSharedTravel && preferredTravelUnit) { + if (passengerIntent.mode === 'shared' && preferredTravelUnit) { travelProfile.sharedTravelUnit = buildSharedTravelUnitContext(preferredTravelUnit); } } @@ -1373,9 +1516,39 @@ export default function ConverseScreen() { // No profile stored — proceed without } + if (!profile) { + passengerIntent = inferPassengerIntent({ + text: displayText, + bookingMode, + preferredTravelUnit, + familyMembers: [], + }); + } + + const planningTranscript = `${basePlanningTranscript}${buildPassengerIntentPlanningNote(passengerIntent)}`; + const usingSharedTravel = passengerIntent.mode === 'shared' && passengerIntent.travellers.length > 1; + + logConverseEvent({ + event: 'plan_requested', + metadata: { + bookingMode: usingSharedTravel ? 'shared' : 'solo', + textLength: displayText.length, + usedFollowUpContext: !!followUpContext, + usedLocationAssumption: !!prepared.assumptionNote, + travellerCount: passengerIntent.travellers.length, + }, + }); + if (!travelProfile && usingSharedTravel && preferredTravelUnit) { travelProfile = { sharedTravelUnit: buildSharedTravelUnitContext(preferredTravelUnit), + requestedTravellers: passengerIntent.travellers, + requestedTravelMode: passengerIntent.mode, + }; + } else if (!travelProfile) { + travelProfile = { + requestedTravellers: passengerIntent.travellers, + requestedTravelMode: passengerIntent.mode, }; } @@ -2363,6 +2536,12 @@ export default function ConverseScreen() { // Last route suggestion — "Same route as last time?" const recentTurns = turns.filter((turn) => turn.role === 'meridian').slice(-4); + const voiceFirstIdle = turns.length === 0 && voiceEnabled && !textFallbackVisible && (phase === 'idle' || phase === 'listening'); + const showActiveTripCard = !!activeTrip; + const showUsualRouteCard = !activeTrip && !!usualRoute; + const showMemoryCard = !activeTrip && !usualRoute && !!routeMemory && !!memorySuggestion; + const showSharedUnitCard = !activeTrip && !usualRoute && !routeMemory && !!preferredTravelUnit; + const showTravelModeCard = !activeTrip && !usualRoute && !routeMemory && !preferredTravelUnit && !!travelModeReminder; return ( @@ -2447,7 +2626,7 @@ export default function ConverseScreen() { )} - {activeTrip && ( + {showActiveTripCard && ( { router.push({ @@ -2486,7 +2665,7 @@ export default function ConverseScreen() { })()} )} - {usualRoute && ( + {showUsualRouteCard && ( { void runIntentWithUiFallback(`${usualRoute.origin} to ${usualRoute.destination}`); }} style={styles.usualRouteCard} @@ -2502,7 +2681,7 @@ export default function ConverseScreen() { )} )} - {!usualRoute && routeMemory && memorySuggestion && ( + {showMemoryCard && ( { void runIntentWithUiFallback(memorySuggestion); }} style={styles.usualRouteCard} @@ -2518,7 +2697,7 @@ export default function ConverseScreen() { )} )} - {preferredTravelUnit && ( + {showSharedUnitCard && ( @@ -2546,7 +2725,7 @@ export default function ConverseScreen() { )} - {travelModeReminder && ( + {travelModeReminder && (!voiceFirstIdle || showTravelModeCard) && ( {travelModeReminder.eyebrow} @@ -2571,12 +2750,14 @@ export default function ConverseScreen() { )} + {!voiceFirstIdle && ( router.push('/(main)/trips')} style={styles.myTripsBtn}> Journeys - {primarySuggestion && ( + )} + {primarySuggestion && !voiceFirstIdle && ( Try this @@ -2681,8 +2862,11 @@ export default function ConverseScreen() { const sym = pendingPlanRef.current?.fiatSymbol ?? currencySymbol; const code = pendingPlanRef.current?.fiatCode ?? currencyCode; const plan = pendingPlanRef.current?.plan ?? []; - // Family members from Phase 1 profile (non-sensitive — names + relationships only) - const tpFam = (pendingPlanRef.current?.travelProfile?.familyMembers as Array<{ name: string; relationship: string; railcard?: string }> | undefined) ?? []; + const requestedTravellers = ((pendingPlanRef.current?.travelProfile?.requestedTravellers as Array<{ + name: string; + relationship: 'adult' | 'child' | 'infant'; + }> | undefined) ?? []) + .filter((traveller) => traveller.name?.trim()); const sharedMembers = ((pendingPlanRef.current?.travelProfile?.sharedTravelUnit as { members?: Array<{ name: string; role: 'self' | 'partner' | 'adult' | 'child' | 'infant' }>; } | undefined)?.members ?? []) @@ -2691,11 +2875,11 @@ export default function ConverseScreen() { name: member.role === 'self' ? 'You' : member.name, relationship: member.role === 'child' || member.role === 'infant' ? member.role : 'adult' as const, })); - const passengers = sharedMembers.length > 0 - ? sharedMembers - : tpFam.length > 0 - ? [{ name: 'You', relationship: 'adult' as const }, ...tpFam] - : null; + const passengers = requestedTravellers.length > 0 + ? requestedTravellers + : sharedMembers.length > 0 + ? sharedMembers + : [{ name: 'You', relationship: 'adult' as const }]; const adultCount = passengers?.filter((p) => p.relationship !== 'child' && p.relationship !== 'infant').length ?? 1; const childCount = passengers?.filter((p) => p.relationship === 'child').length ?? 0; const travellerCount = passengers?.length ?? 1; diff --git a/apps/meridian/app/(main)/travel-together.tsx b/apps/meridian/app/(main)/travel-together.tsx index c7cd597..542fa67 100644 --- a/apps/meridian/app/(main)/travel-together.tsx +++ b/apps/meridian/app/(main)/travel-together.tsx @@ -14,6 +14,7 @@ import { router, useLocalSearchParams } from 'expo-router'; import { Ionicons } from '@expo/vector-icons'; import * as Linking from 'expo-linking'; import { C } from '../../lib/theme'; +import { loadProfileRaw, type TravelProfile } from '../../lib/profile'; import { loadCredentials } from '../../lib/storage'; import { acceptSharedTravelInvite, createSharedTravelInvite, listSharedTravelUnits, type SharedTravelRemoteUnit } from '../../lib/api'; import { @@ -96,6 +97,42 @@ function inviteQrUrl(inviteToken: string) { return `https://api.qrserver.com/v1/create-qr-code/?size=360x360&data=${encodeURIComponent(url)}&format=png&margin=2`; } +function firstName(name: string): string { + return name.trim().split(/\s+/)[0] ?? name.trim(); +} + +function buildTravelUnitFromProfile(profile: TravelProfile): TravelUnit | null { + const familyMembers = (profile.familyMembers ?? []).filter((member) => member.name.trim()); + if (familyMembers.length === 0) return null; + + const selfMember = makeBlankMember('self'); + selfMember.name = firstName(profile.legalName) || 'You'; + + const members = familyMembers.map((member) => { + const role: TravelUnitMemberRole = + member.relationship === 'adult' + ? familyMembers.length === 1 ? 'partner' : 'adult' + : member.relationship; + return { + ...makeBlankMember(role), + name: member.name.trim(), + role, + state: 'guest' as const, + }; + }); + + return { + id: makeTravelUnitId(), + name: familyMembers.length === 1 ? `${selfMember.name} & ${firstName(familyMembers[0].name)}` : 'Family', + type: familyMembers.length === 1 ? 'couple' : 'family', + members: [selfMember, ...members], + primaryPayerMemberId: selfMember.id, + notes: '', + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }; +} + export default function TravelTogetherScreen() { const params = useLocalSearchParams<{ invite?: string }>(); const [units, setUnits] = useState([]); @@ -109,14 +146,16 @@ export default function TravelTogetherScreen() { const [remoteNotice, setRemoteNotice] = useState(null); const [remoteBusy, setRemoteBusy] = useState(false); const [remoteUnits, setRemoteUnits] = useState([]); + const [profile, setProfile] = useState(null); const refresh = useCallback(() => { - void Promise.all([loadTravelUnits(), loadPreferredTravelUnitId(), loadCredentials()]) - .then(async ([loadedUnits, loadedPreferredId, creds]) => { + void Promise.all([loadTravelUnits(), loadPreferredTravelUnitId(), loadCredentials(), loadProfileRaw().catch(() => null)]) + .then(async ([loadedUnits, loadedPreferredId, creds, loadedProfile]) => { const nextUnits = loadedUnits.length > 0 ? loadedUnits : [makeBlankUnit()]; setUnits(nextUnits); setPreferredId(loadedPreferredId); setAgentCreds(creds); + setProfile(loadedProfile); setSelectedId((current) => current && nextUnits.some((unit) => unit.id === current) ? current : nextUnits[0]?.id ?? null); if (creds) { @@ -164,6 +203,36 @@ export default function TravelTogetherScreen() { setSelectedId(nextUnit.id); }, []); + const savedCompanions = (profile?.familyMembers ?? []).filter((member) => member.name.trim()); + + const handleImportSavedCompanions = useCallback(async () => { + if (!profile) { + Alert.alert('Profile not ready', 'Save your family or companion details first so Ace can import them here.'); + return; + } + + const importedUnit = buildTravelUnitFromProfile(profile); + if (!importedUnit) { + Alert.alert('No saved companions yet', 'Add a partner, child, or family member to your profile first.'); + return; + } + + setSaving(true); + try { + await saveTravelUnit(importedUnit); + await savePreferredTravelUnitId(importedUnit.id); + replaceSelectedUnit(importedUnit); + setPreferredId(importedUnit.id); + setRemoteNotice( + importedUnit.type === 'couple' + ? `${importedUnit.name} is now your usual shared travel unit.` + : 'Your saved family is now ready as a shared travel unit.', + ); + } finally { + setSaving(false); + } + }, [profile, replaceSelectedUnit]); + const handleSave = useCallback(async () => { if (!selectedUnit) return; @@ -411,6 +480,22 @@ export default function TravelTogetherScreen() { + {savedCompanions.length > 0 && ( + + + + Bring in saved companions + + Ace already knows {savedCompanions.length === 1 ? `${savedCompanions[0].name} from your profile` : `${savedCompanions.length} saved companions from your profile`}. Import them once so shared booking and natural voice phrasing work cleanly. + + + { void handleImportSavedCompanions(); }} style={styles.importBtn} disabled={saving}> + {saving ? 'Working...' : 'Import'} + + + + )} + Link another Ace account @@ -741,6 +826,13 @@ const styles = StyleSheet.create({ infoCard: { backgroundColor: C.surface2, borderWidth: 1, borderColor: C.borderMd, borderRadius: 14, padding: 14, marginBottom: 18 }, infoTitle: { fontSize: 13, fontWeight: '700', color: C.textPrimary, marginBottom: 6 }, infoBody: { fontSize: 12, color: C.textSecondary, lineHeight: 18 }, + importCard: { backgroundColor: '#0a1624', borderWidth: 1, borderColor: '#17314a', borderRadius: 14, padding: 14, marginBottom: 18 }, + importHeader: { flexDirection: 'row', alignItems: 'center', gap: 12 }, + importCopy: { flex: 1 }, + importTitle: { fontSize: 13, fontWeight: '700', color: '#edf6ff', marginBottom: 6 }, + importBody: { fontSize: 12, color: '#a9bfd4', lineHeight: 18 }, + importBtn: { borderRadius: 12, backgroundColor: '#1d4ed8', paddingHorizontal: 14, paddingVertical: 10, alignItems: 'center', justifyContent: 'center' }, + importBtnText: { fontSize: 12, fontWeight: '700', color: '#eff6ff' }, linkCard: { backgroundColor: '#120914', borderWidth: 1, borderColor: '#4a1d67', borderRadius: 14, padding: 14, marginBottom: 18 }, linkTitle: { fontSize: 13, fontWeight: '700', color: '#f5d0fe', marginBottom: 6 }, linkBody: { fontSize: 12, color: '#e9d5ff', lineHeight: 18, marginBottom: 12 }, From 28a75baa5be697dbc6018268c8ef98b9175cc4bd Mon Sep 17 00:00:00 2001 From: Baskaran Balasubramanian Date: Fri, 10 Apr 2026 10:11:03 +0530 Subject: [PATCH 08/10] tighten ace party scope and payment seams --- .codex/hooks.json | 15 ++++++++++++ AGENTS.md | 9 +++++++ apps/api-edge/src/routes/concierge.ts | 30 ++++++++++++++++++++--- apps/api-edge/src/routes/paymentsSetup.ts | 13 ++++++++-- apps/meridian/app/(main)/converse.tsx | 9 ++++++- apps/meridian/app/(main)/wallet.tsx | 17 +++++++------ apps/meridian/lib/api.ts | 6 +++++ 7 files changed, 84 insertions(+), 15 deletions(-) create mode 100644 .codex/hooks.json diff --git a/.codex/hooks.json b/.codex/hooks.json new file mode 100644 index 0000000..aa3df6b --- /dev/null +++ b/.codex/hooks.json @@ -0,0 +1,15 @@ +{ + "hooks": { + "PreToolUse": [ + { + "matcher": "Bash", + "hooks": [ + { + "type": "command", + "command": "[ -f graphify-out/graph.json ] && echo '{\"hookSpecificOutput\":{\"hookEventName\":\"PreToolUse\",\"permissionDecision\":\"allow\"},\"systemMessage\":\"graphify: Knowledge graph exists. Read graphify-out/GRAPH_REPORT.md for god nodes and community structure before searching raw files.\"}' || true" + } + ] + } + ] + } +} \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 41341a3..baebd03 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -155,3 +155,12 @@ If there is tension between: - and a cleaner product seam choose the cleaner seam. + +## graphify + +If `graphify-out/` exists for this project, treat it as the preferred map before falling back to raw-file search. + +Rules: +- Before answering architecture or codebase questions, read `graphify-out/GRAPH_REPORT.md` when it exists for god nodes and community structure. +- If `graphify-out/wiki/index.md` exists, navigate it instead of reading raw files. +- After modifying code files, refresh the graph only when graphify output already exists or graphify is actively being used for this repo. diff --git a/apps/api-edge/src/routes/concierge.ts b/apps/api-edge/src/routes/concierge.ts index cd8c5aa..b7f86fb 100644 --- a/apps/api-edge/src/routes/concierge.ts +++ b/apps/api-edge/src/routes/concierge.ts @@ -572,11 +572,19 @@ conciergeRouter.post('/intent', async (c) => { : ''; // ── Family / group context ──────────────────────────────────────────────── + const requestedTravellers = travelProfile?.requestedTravellers as Array<{ + name: string; + relationship: 'adult' | 'child' | 'infant'; + }> | undefined; const familyMembers = travelProfile?.familyMembers as Array<{ id: string; name: string; relationship: string; dateOfBirth?: string; railcard?: string; documentNumber?: string; documentExpiry?: string; nationality?: string; }> | undefined; + const requestedTravellerContext = requestedTravellers && requestedTravellers.length > 0 + ? `\nRequested party for this trip: ${requestedTravellers.map((traveller) => `${traveller.name} (${traveller.relationship})`).join('; ')}. Only plan and price for this party. Do not silently expand to every saved family member unless the customer explicitly asked for that larger group.` + : '\nRequested party for this trip: device owner only unless the customer explicitly asked for a companion or shared trip.'; + const familyContext = familyMembers && familyMembers.length > 0 ? (() => { const lines = familyMembers.map(m => { @@ -593,11 +601,11 @@ conciergeRouter.post('/intent', async (c) => { const adultCount = 1 + familyMembers.filter(m => m.relationship === 'adult').length; const childCount = familyMembers.filter(m => m.relationship === 'child').length; const hasFamilyRailcard = adultCount >= 2 && childCount >= 1 && childCount <= 4; - return `\nUser's family: ${lines.join('; ')}.${hasFamilyRailcard ? ' Family & Friends Railcard applies — 1/3 off adult fares, 60% off child fares. Apply automatically and mention the saving.' : ''}`; + return `\nSaved family memory: ${lines.join('; ')}.${hasFamilyRailcard ? ' Family & Friends Railcard applies — 1/3 off adult fares, 60% off child fares when the requested party qualifies. Apply it only when the active travellers fit.' : ''}`; })() : ''; - const systemPrompt = `You are Ace — a travel fixer, not an assistant.${locationContext}${nationalityContext}${railcardContext}${indiaClassContext}${subscriptionContext}${familyContext} + const systemPrompt = `You are Ace — a travel fixer, not an assistant.${locationContext}${nationalityContext}${railcardContext}${indiaClassContext}${subscriptionContext}${requestedTravellerContext}${familyContext} You've worked every booking desk on earth and left. You know UK railcards, IRCTC tatkal quotas, off-peak windows, coach classes, waitlists. You get things done quietly and tell people after. CHARACTER: @@ -809,12 +817,26 @@ PHASE 2 CONFIRMATION FORMAT (when hire result arrives): }]; } - // Add family members as additional passengers + const requestedParty = travelProfile?.requestedTravellers as Array<{ + name: string; + relationship: 'adult' | 'child' | 'infant'; + }> | undefined; + + const requestedAdditionalNames = new Set( + (requestedParty ?? []) + .map((traveller) => traveller.name?.trim().toLowerCase()) + .filter((name) => name && name !== 'you'), + ); + + // Add only the explicitly requested saved family members as additional passengers const familyMems = travelProfile?.familyMembers as Array<{ name: string; relationship: string; dateOfBirth?: string; documentNumber?: string; documentExpiry?: string; nationality?: string; }> | undefined; - const additionalPassengers: DuffelPassenger[] = (familyMems ?? []).map((m) => { + const selectedFamilyMems = requestedAdditionalNames.size > 0 + ? (familyMems ?? []).filter((member) => requestedAdditionalNames.has(member.name.trim().toLowerCase())) + : []; + const additionalPassengers: DuffelPassenger[] = selectedFamilyMems.map((m) => { const mParts = m.name.trim().split(/\s+/); const p: DuffelPassenger = { given_name: mParts[0] ?? m.name, diff --git a/apps/api-edge/src/routes/paymentsSetup.ts b/apps/api-edge/src/routes/paymentsSetup.ts index 953ebba..be00b2a 100644 --- a/apps/api-edge/src/routes/paymentsSetup.ts +++ b/apps/api-edge/src/routes/paymentsSetup.ts @@ -11,7 +11,9 @@ * GET /api/payments/methods/:principalId — list saved payment methods * DELETE /api/payments/methods/:methodId — remove a payment method * - * Auth: all routes require a valid merchant API key (authenticateApiKey). + * Auth: + * - first-party Ace app via x-bro-key + * - merchant API key via authenticateApiKey * * Flow: * 1. Client calls POST /setup-intent → gets clientSecret @@ -100,7 +102,14 @@ async function ensureStripeCustomer(stripe: Stripe, sql: ReturnType(); -router.use('*', authenticateApiKey); +router.use('*', async (c, next) => { + const broKey = c.req.header('x-bro-key') ?? ''; + if (c.env.BRO_CLIENT_KEY && broKey === c.env.BRO_CLIENT_KEY) { + await next(); + return; + } + return authenticateApiKey(c, next); +}); // --------------------------------------------------------------------------- // POST /setup-intent — Create a Stripe Setup Intent diff --git a/apps/meridian/app/(main)/converse.tsx b/apps/meridian/app/(main)/converse.tsx index 0351674..cf058eb 100644 --- a/apps/meridian/app/(main)/converse.tsx +++ b/apps/meridian/app/(main)/converse.tsx @@ -2011,7 +2011,14 @@ export default function ConverseScreen() { // This is the first time legalName, email, phone, and documents enter memory. try { const fullProfile = await loadProfileAuthenticated(); - if (fullProfile) pending.fullProfile = fullProfile as unknown as Record; + if (fullProfile) { + pending.fullProfile = { + ...(fullProfile as unknown as Record), + requestedTravellers: pending.travelProfile?.requestedTravellers, + requestedTravelMode: pending.travelProfile?.requestedTravelMode, + sharedTravelUnit: pending.travelProfile?.sharedTravelUnit, + }; + } } catch { // Biometric already succeeded — proceed without full profile if load fails } diff --git a/apps/meridian/app/(main)/wallet.tsx b/apps/meridian/app/(main)/wallet.tsx index 62930b5..515c36d 100644 --- a/apps/meridian/app/(main)/wallet.tsx +++ b/apps/meridian/app/(main)/wallet.tsx @@ -157,11 +157,12 @@ export default function WalletScreen() { ))} - {/* Add card — opens Setup Intent flow once @stripe/stripe-react-native is installed */} - Alert.alert('Add Card', 'Complete Stripe setup to enable native card entry.\nInstall @stripe/stripe-react-native and wire createSetupIntent() from lib/api.ts.')}> - - Add payment card - + + + + Saved cards are being finished for this device. For now, keep your Ace balance topped up and bookings can still move cleanly. + + {/* Top-up instructions */} @@ -248,16 +249,16 @@ const styles = StyleSheet.create({ }, cardLabel: { fontSize: 14, color: '#d1d5db', fontWeight: '500' }, defaultBadge: { fontSize: 11, color: '#6366f1', marginTop: 2 }, - addCardBtn: { + addCardNotice: { flexDirection: 'row', - alignItems: 'center', + alignItems: 'flex-start', gap: 8, paddingTop: 12, marginTop: 4, borderTopWidth: 1, borderTopColor: '#1a1a1a', }, - addCardText: { fontSize: 14, color: '#6366f1' }, + addCardNoticeText: { flex: 1, fontSize: 13, lineHeight: 18, color: '#8ea4bc' }, depositCard: { backgroundColor: '#111111', borderRadius: 14, diff --git a/apps/meridian/lib/api.ts b/apps/meridian/lib/api.ts index dd2cf83..cc92688 100644 --- a/apps/meridian/lib/api.ts +++ b/apps/meridian/lib/api.ts @@ -11,6 +11,7 @@ const BRO_KEY = process.env.EXPO_PUBLIC_BRO_KEY ?? ''; function requiresBroKey(path: string): boolean { return ( path.startsWith('/api/concierge/') || + path.startsWith('/api/payments/') || path.startsWith('/api/shared-travel/') || path.startsWith('/api/trip-rooms/') || path.startsWith('/api/support/') @@ -92,6 +93,11 @@ export interface BroTravelProfile { currentLat?: number; currentLon?: number; familyMembers?: BroFamilyMember[]; + requestedTravellers?: Array<{ + name: string; + relationship: 'adult' | 'child' | 'infant'; + }>; + requestedTravelMode?: 'solo' | 'shared'; sharedTravelUnit?: { id: string; name: string; From a40068c1547050758ed3536a3ff60d9fccc2b3ac Mon Sep 17 00:00:00 2001 From: Baskaran Balasubramanian Date: Fri, 10 Apr 2026 16:00:36 +0530 Subject: [PATCH 09/10] fix(ace): harden premium voice startup --- apps/meridian/app/(main)/converse.tsx | 56 +++++++++++++++------------ apps/meridian/lib/tts.ts | 24 +++++++++--- apps/meridian/lib/voiceSession.ts | 8 ++-- 3 files changed, 54 insertions(+), 34 deletions(-) diff --git a/apps/meridian/app/(main)/converse.tsx b/apps/meridian/app/(main)/converse.tsx index cf058eb..0f59423 100644 --- a/apps/meridian/app/(main)/converse.tsx +++ b/apps/meridian/app/(main)/converse.tsx @@ -1398,6 +1398,26 @@ export default function ConverseScreen() { }, 80); }, [activeTrip, clearHandsFreeListenTimer, setPhase, voiceEnabled]); + const openTextFallback = useCallback((message: string, seed?: string) => { + clearHandsFreeListenTimer(); + cancelSpeech(); + setIsSpeaking(false); + ttsAmplitude.value = 0; + if (phaseRef.current === 'listening') { + void stopRecording().catch(() => null); + recordingActiveRef.current = false; + startingRecordingRef.current = false; + micAmplitude.value = 0; + phaseRef.current = 'idle'; + setPhase('idle'); + } + setError(message); + setTextFallbackVisible(true); + if (seed != null) { + setTextFallbackDraft(seed); + } + }, [clearHandsFreeListenTimer, setError, setPhase]); + const speakIfEnabled = useCallback(async (text: string, restartListening = true) => { if (!voiceEnabled) return; clearHandsFreeListenTimer(); @@ -1415,6 +1435,12 @@ export default function ConverseScreen() { setIsSpeaking(false); ttsAmplitude.value = 0; } + if (!speakingStarted) { + openTextFallback( + 'Ace voice is unavailable right now. Type the trip once below and Ace will keep the journey moving.', + ); + return; + } if (restartListening && !keyboardVisibleRef.current && !textFallbackVisibleRef.current) { const cur = phaseRef.current; // Don't auto-restart when a booking card is showing, payment is processing, or booking is done @@ -1422,27 +1448,7 @@ export default function ConverseScreen() { armHandsFreeListen(1600); } } - }, [armHandsFreeListen, clearHandsFreeListenTimer, voiceEnabled]); - - const openTextFallback = useCallback((message: string, seed?: string) => { - clearHandsFreeListenTimer(); - cancelSpeech(); - setIsSpeaking(false); - ttsAmplitude.value = 0; - if (phaseRef.current === 'listening') { - void stopRecording().catch(() => null); - recordingActiveRef.current = false; - startingRecordingRef.current = false; - micAmplitude.value = 0; - phaseRef.current = 'idle'; - setPhase('idle'); - } - setError(message); - setTextFallbackVisible(true); - if (seed != null) { - setTextFallbackDraft(seed); - } - }, [clearHandsFreeListenTimer, setError, setPhase]); + }, [armHandsFreeListen, clearHandsFreeListenTimer, openTextFallback, voiceEnabled]); // ── Phase 1: plan ───────────────────────────────────────────────────────── @@ -2135,10 +2141,10 @@ export default function ConverseScreen() { }); let nextMessage = __DEV__ ? `STT: ${rawMessage}` : 'Voice error — try again.'; if (!__DEV__) { - nextMessage = 'Ace missed that. Say it again, or type the trip below.'; + nextMessage = 'I did not catch the trip clearly. Say it once more, or type it below.'; } if (msg.includes('timed out') || msg.includes('timeout') || msg.includes('abort')) { - nextMessage = 'Ace did not catch that in time. Say it again, or type it below.'; + nextMessage = 'I did not catch that in time. Say it once more, or type it below.'; } else if (msg.includes('401') || msg.includes('403') || msg.includes('not authorised')) { nextMessage = 'Voice service is unavailable right now. You can type the trip below.'; } else if (msg.includes('503') || msg.includes('not configured')) { @@ -2212,7 +2218,7 @@ export default function ConverseScreen() { message: rawMessage, metadata: { phase: 'capture' }, }); - let nextMessage = 'Ace missed that. Say it again, or type the trip below.'; + let nextMessage = 'I did not catch the trip clearly. Say it once more, or type it below.'; if (msg.includes('timed out') || msg.includes('timeout')) { nextMessage = 'Ace took too long on that request. Try again, or type it below.'; } else if (msg.includes('no connection') || msg.includes('internet') || msg.includes('network')) { @@ -2429,7 +2435,7 @@ export default function ConverseScreen() { phase === 'listening' ? (liveVoiceReady ? 'Speak naturally. Ace will stay with the thread.' : 'Say the trip once, naturally.') : phase === 'thinking' || phase === 'hiring' || phase === 'executing' ? 'Ace is working the route, timing, and booking.' : phase === 'done' ? 'Trip secured. Ace will stay with it.' : - phase === 'error' ? 'Say it again or continue in text below.' : + phase === 'error' ? 'Say it once more, or continue in text below.' : voiceEnabled ? (liveVoiceReady ? 'Ace is live. Say the trip when you are ready.' : 'Say the trip when you are ready.') : 'Tap Ace to resume voice.'; const presenceTone = diff --git a/apps/meridian/lib/tts.ts b/apps/meridian/lib/tts.ts index c9b1621..1311ade 100644 --- a/apps/meridian/lib/tts.ts +++ b/apps/meridian/lib/tts.ts @@ -6,6 +6,7 @@ import { trackClientEvent } from './telemetry'; const BASE = process.env.EXPO_PUBLIC_API_URL ?? 'https://api.agentpay.so'; const BRO_KEY = process.env.EXPO_PUBLIC_BRO_KEY ?? ''; +const PLAYBACK_START_TIMEOUT_MS = Platform.OS === 'ios' ? 4000 : 2000; let activeSound: Audio.Sound | null = null; let activeUri: string | null = null; @@ -266,7 +267,7 @@ export async function speakBro(text: string, options?: SpeakBroOptions): Promise const { sound } = await Audio.Sound.createAsync( { uri }, { - shouldPlay: true, + shouldPlay: false, volume: 1.0, androidImplementation: 'MediaPlayer', }, @@ -302,8 +303,9 @@ export async function speakBro(text: string, options?: SpeakBroOptions): Promise } }; - // Watchdog: audio must start within 1200ms of sound creation. - // Flash model files are larger than Turbo; 600ms was too tight. + // Give iOS more time to bind the route and begin playback after the + // file is written to cache. If Ace still has not started by then, + // surface it as a real failure instead of resolving silently. const watchdog = setTimeout(() => { if (!playbackStarted && !settled) { settled = true; @@ -311,10 +313,10 @@ export async function speakBro(text: string, options?: SpeakBroOptions): Promise try { sound.setOnAudioSampleReceived(null); } catch {} options?.onMeter?.(0); void stopActivePlayback().finally(() => - reject(new Error('TTS playback watchdog: no audio start within 1200ms')), + reject(new Error(`TTS playback watchdog: no audio start within ${PLAYBACK_START_TIMEOUT_MS}ms`)), ); } - }, 1200); + }, PLAYBACK_START_TIMEOUT_MS); sound.setOnPlaybackStatusUpdate((status) => { if (status.isLoaded && status.isPlaying && !playbackStarted) { @@ -334,6 +336,18 @@ export async function speakBro(text: string, options?: SpeakBroOptions): Promise } } }); + + void sound.playAsync().catch((error) => { + clearTimeout(watchdog); + if (settled) return; + settled = true; + stopMeterPulse(); + try { sound.setOnAudioSampleReceived(null); } catch {} + options?.onMeter?.(0); + void stopActivePlayback().finally(() => + reject(error instanceof Error ? error : new Error(String(error))), + ); + }); }); void trackClientEvent({ event: 'tts_played', diff --git a/apps/meridian/lib/voiceSession.ts b/apps/meridian/lib/voiceSession.ts index 5869437..307a9da 100644 --- a/apps/meridian/lib/voiceSession.ts +++ b/apps/meridian/lib/voiceSession.ts @@ -4,17 +4,17 @@ export const DEFAULT_VOICE_SESSION_CONFIG: VoiceSessionConfig = { mode: 'batch', transport: 'http', provider: 'batch_proxy', - ready: false, + ready: true, planningToolsAvailableDuringConversation: true, bookingToolsLockedUntilConfirm: true, supportsInterruptions: false, supportsServerVad: false, - premiumVoice: 'none', + premiumVoice: 'elevenlabs', fallback: { stt: 'whisper_proxy', - tts: 'none', + tts: 'elevenlabs_http', }, - diagnostics: ['session_manifest_unavailable'], + diagnostics: ['session_manifest_unavailable_batch_retained'], }; let cachedVoiceSessionConfig: VoiceSessionConfig | null = null; From 848198a8844fd00c4a9edcb02982c2237d3ad1c0 Mon Sep 17 00:00:00 2001 From: Baskaran Balasubramanian Date: Fri, 10 Apr 2026 17:05:27 +0530 Subject: [PATCH 10/10] fix(ios): add camera privacy purpose string --- apps/meridian/app.json | 1 + apps/meridian/ios/Ace/Info.plist | 97 ++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 apps/meridian/ios/Ace/Info.plist diff --git a/apps/meridian/app.json b/apps/meridian/app.json index 8903c2e..50eae82 100644 --- a/apps/meridian/app.json +++ b/apps/meridian/app.json @@ -76,6 +76,7 @@ "infoPlist": { "NSMicrophoneUsageDescription": "Ace uses your microphone to capture voice requests.", "NSSpeechRecognitionUsageDescription": "Ace uses speech recognition to understand your requests.", + "NSCameraUsageDescription": "Ace uses the camera only when you add travel documents or scan booking details.", "NSFaceIDUsageDescription": "Ace uses Face ID to protect your stored credentials.", "NSLocationWhenInUseUsageDescription": "Ace uses your location to find your nearest station and suggest quick routes.", "ITSAppUsesNonExemptEncryption": false diff --git a/apps/meridian/ios/Ace/Info.plist b/apps/meridian/ios/Ace/Info.plist new file mode 100644 index 0000000..8ba0f87 --- /dev/null +++ b/apps/meridian/ios/Ace/Info.plist @@ -0,0 +1,97 @@ + + + + + CADisableMinimumFrameDurationOnPhone + + CFBundleDevelopmentRegion + $(DEVELOPMENT_LANGUAGE) + CFBundleDisplayName + Ace + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + $(PRODUCT_BUNDLE_PACKAGE_TYPE) + CFBundleShortVersionString + 1.0.0 + CFBundleSignature + ???? + CFBundleURLTypes + + + CFBundleURLSchemes + + meridian + so.agentpay.meridian + + + + CFBundleVersion + 11 + ITSAppUsesNonExemptEncryption + + LSMinimumSystemVersion + 12.0 + LSRequiresIPhoneOS + + NSAppTransportSecurity + + NSAllowsArbitraryLoads + + NSAllowsLocalNetworking + + + NSFaceIDUsageDescription + Ace uses Face ID to protect your stored credentials. + NSCameraUsageDescription + Ace uses the camera only when you add travel documents or scan booking details. + NSLocationAlwaysAndWhenInUseUsageDescription + Ace uses your location to find your nearest station. + NSLocationAlwaysUsageDescription + Allow $(PRODUCT_NAME) to access your location + NSLocationWhenInUseUsageDescription + Ace uses your location to find your nearest station and suggest quick routes. + NSMicrophoneUsageDescription + Ace uses your microphone to capture voice requests. + NSMotionUsageDescription + Allow $(PRODUCT_NAME) to access your device motion + NSSpeechRecognitionUsageDescription + Ace uses speech recognition to understand your requests. + NSUserActivityTypes + + $(PRODUCT_BUNDLE_IDENTIFIER).expo.index_route + + UILaunchStoryboardName + SplashScreen + UIRequiredDeviceCapabilities + + arm64 + + UIRequiresFullScreen + + UIStatusBarStyle + UIStatusBarStyleDefault + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + UIUserInterfaceStyle + Dark + UIViewControllerBasedStatusBarAppearance + + +