diff --git a/.codex/hooks.json b/.codex/hooks.json new file mode 100644 index 0000000..aa3df6b --- /dev/null +++ b/.codex/hooks.json @@ -0,0 +1,15 @@ +{ + "hooks": { + "PreToolUse": [ + { + "matcher": "Bash", + "hooks": [ + { + "type": "command", + "command": "[ -f graphify-out/graph.json ] && echo '{\"hookSpecificOutput\":{\"hookEventName\":\"PreToolUse\",\"permissionDecision\":\"allow\"},\"systemMessage\":\"graphify: Knowledge graph exists. Read graphify-out/GRAPH_REPORT.md for god nodes and community structure before searching raw files.\"}' || true" + } + ] + } + ] + } +} \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 41341a3..baebd03 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -155,3 +155,12 @@ If there is tension between: - and a cleaner product seam choose the cleaner seam. + +## graphify + +If `graphify-out/` exists for this project, treat it as the preferred map before falling back to raw-file search. + +Rules: +- Before answering architecture or codebase questions, read `graphify-out/GRAPH_REPORT.md` when it exists for god nodes and community structure. +- If `graphify-out/wiki/index.md` exists, navigate it instead of reading raw files. +- After modifying code files, refresh the graph only when graphify output already exists or graphify is actively being used for this repo. diff --git a/apps/api-edge/src/routes/concierge.ts b/apps/api-edge/src/routes/concierge.ts index cd8c5aa..b7f86fb 100644 --- a/apps/api-edge/src/routes/concierge.ts +++ b/apps/api-edge/src/routes/concierge.ts @@ -572,11 +572,19 @@ conciergeRouter.post('/intent', async (c) => { : ''; // ── Family / group context ──────────────────────────────────────────────── + const requestedTravellers = travelProfile?.requestedTravellers as Array<{ + name: string; + relationship: 'adult' | 'child' | 'infant'; + }> | undefined; const familyMembers = travelProfile?.familyMembers as Array<{ id: string; name: string; relationship: string; dateOfBirth?: string; railcard?: string; documentNumber?: string; documentExpiry?: string; nationality?: string; }> | undefined; + const requestedTravellerContext = requestedTravellers && requestedTravellers.length > 0 + ? `\nRequested party for this trip: ${requestedTravellers.map((traveller) => `${traveller.name} (${traveller.relationship})`).join('; ')}. Only plan and price for this party. Do not silently expand to every saved family member unless the customer explicitly asked for that larger group.` + : '\nRequested party for this trip: device owner only unless the customer explicitly asked for a companion or shared trip.'; + const familyContext = familyMembers && familyMembers.length > 0 ? (() => { const lines = familyMembers.map(m => { @@ -593,11 +601,11 @@ conciergeRouter.post('/intent', async (c) => { const adultCount = 1 + familyMembers.filter(m => m.relationship === 'adult').length; const childCount = familyMembers.filter(m => m.relationship === 'child').length; const hasFamilyRailcard = adultCount >= 2 && childCount >= 1 && childCount <= 4; - return `\nUser's family: ${lines.join('; ')}.${hasFamilyRailcard ? ' Family & Friends Railcard applies — 1/3 off adult fares, 60% off child fares. Apply automatically and mention the saving.' : ''}`; + return `\nSaved family memory: ${lines.join('; ')}.${hasFamilyRailcard ? ' Family & Friends Railcard applies — 1/3 off adult fares, 60% off child fares when the requested party qualifies. Apply it only when the active travellers fit.' : ''}`; })() : ''; - const systemPrompt = `You are Ace — a travel fixer, not an assistant.${locationContext}${nationalityContext}${railcardContext}${indiaClassContext}${subscriptionContext}${familyContext} + const systemPrompt = `You are Ace — a travel fixer, not an assistant.${locationContext}${nationalityContext}${railcardContext}${indiaClassContext}${subscriptionContext}${requestedTravellerContext}${familyContext} You've worked every booking desk on earth and left. You know UK railcards, IRCTC tatkal quotas, off-peak windows, coach classes, waitlists. You get things done quietly and tell people after. CHARACTER: @@ -809,12 +817,26 @@ PHASE 2 CONFIRMATION FORMAT (when hire result arrives): }]; } - // Add family members as additional passengers + const requestedParty = travelProfile?.requestedTravellers as Array<{ + name: string; + relationship: 'adult' | 'child' | 'infant'; + }> | undefined; + + const requestedAdditionalNames = new Set( + (requestedParty ?? []) + .map((traveller) => traveller.name?.trim().toLowerCase()) + .filter((name) => name && name !== 'you'), + ); + + // Add only the explicitly requested saved family members as additional passengers const familyMems = travelProfile?.familyMembers as Array<{ name: string; relationship: string; dateOfBirth?: string; documentNumber?: string; documentExpiry?: string; nationality?: string; }> | undefined; - const additionalPassengers: DuffelPassenger[] = (familyMems ?? []).map((m) => { + const selectedFamilyMems = requestedAdditionalNames.size > 0 + ? (familyMems ?? []).filter((member) => requestedAdditionalNames.has(member.name.trim().toLowerCase())) + : []; + const additionalPassengers: DuffelPassenger[] = selectedFamilyMems.map((m) => { const mParts = m.name.trim().split(/\s+/); const p: DuffelPassenger = { given_name: mParts[0] ?? m.name, diff --git a/apps/api-edge/src/routes/paymentsSetup.ts b/apps/api-edge/src/routes/paymentsSetup.ts index 953ebba..be00b2a 100644 --- a/apps/api-edge/src/routes/paymentsSetup.ts +++ b/apps/api-edge/src/routes/paymentsSetup.ts @@ -11,7 +11,9 @@ * GET /api/payments/methods/:principalId — list saved payment methods * DELETE /api/payments/methods/:methodId — remove a payment method * - * Auth: all routes require a valid merchant API key (authenticateApiKey). + * Auth: + * - first-party Ace app via x-bro-key + * - merchant API key via authenticateApiKey * * Flow: * 1. Client calls POST /setup-intent → gets clientSecret @@ -100,7 +102,14 @@ async function ensureStripeCustomer(stripe: Stripe, sql: ReturnType(); -router.use('*', authenticateApiKey); +router.use('*', async (c, next) => { + const broKey = c.req.header('x-bro-key') ?? ''; + if (c.env.BRO_CLIENT_KEY && broKey === c.env.BRO_CLIENT_KEY) { + await next(); + return; + } + return authenticateApiKey(c, next); +}); // --------------------------------------------------------------------------- // POST /setup-intent — Create a Stripe Setup Intent diff --git a/apps/api-edge/src/routes/voice.ts b/apps/api-edge/src/routes/voice.ts index c11155b..f41b627 100644 --- a/apps/api-edge/src/routes/voice.ts +++ b/apps/api-edge/src/routes/voice.ts @@ -17,6 +17,145 @@ export const voiceRouter = new Hono<{ Bindings: Env; Variables: Variables }>(); // Daniel (onwK4e9ZLuTAKqWW03F9) — deeper/more formal, but noticeably slower to first byte. const DEFAULT_ELEVENLABS_VOICE_ID = 'JBFqnCBsd6RMkjVDRZzb'; +type VoiceSessionMode = 'batch' | 'live'; +type VoiceSessionProvider = 'batch_proxy' | 'livekit' | 'openai_realtime'; + +type VoiceSessionConnectResponse = { + mode: 'live'; + provider: 'livekit'; + serverUrl: string; + roomName: string; + participantIdentity: string; + participantToken: string; + expiresAt: string; + planningToolsAvailableDuringConversation: true; + bookingToolsLockedUntilConfirm: true; +}; + +function getLiveVoiceProvider(env: Env): VoiceSessionProvider | null { + if (env.LIVE_VOICE_RUNTIME_ENABLED !== 'true') { + return null; + } + if (env.LIVEKIT_URL && env.LIVEKIT_API_KEY && env.LIVEKIT_API_SECRET) { + return 'livekit'; + } + if (env.OPENAI_API_KEY && env.OPENAI_REALTIME_MODEL) { + return 'openai_realtime'; + } + return null; +} + +function base64UrlEncode(input: string | Uint8Array): string { + const bytes = typeof input === 'string' ? new TextEncoder().encode(input) : input; + let binary = ''; + for (const byte of bytes) binary += String.fromCharCode(byte); + return btoa(binary).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/g, ''); +} + +async function signHs256(secret: string, data: string): Promise { + const key = await crypto.subtle.importKey( + 'raw', + new TextEncoder().encode(secret), + { name: 'HMAC', hash: 'SHA-256' }, + false, + ['sign'], + ); + const signature = await crypto.subtle.sign('HMAC', key, new TextEncoder().encode(data)); + return base64UrlEncode(new Uint8Array(signature)); +} + +function slugPart(value: string, fallback: string): string { + const normalized = value.trim().toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, ''); + return normalized.slice(0, 32) || fallback; +} + +async function createLiveKitParticipantToken(params: { + apiKey: string; + apiSecret: string; + roomName: string; + participantIdentity: string; + metadata?: Record; + ttlSeconds?: number; +}): Promise<{ token: string; expiresAt: string }> { + const now = Math.floor(Date.now() / 1000); + const ttlSeconds = params.ttlSeconds ?? 60 * 60; + const exp = now + ttlSeconds; + const header = { alg: 'HS256', typ: 'JWT' }; + const payload = { + iss: params.apiKey, + sub: params.participantIdentity, + nbf: now - 10, + exp, + video: { + room: params.roomName, + roomJoin: true, + canPublish: true, + canSubscribe: true, + canPublishData: true, + }, + metadata: params.metadata ? JSON.stringify(params.metadata) : undefined, + }; + const encodedHeader = base64UrlEncode(JSON.stringify(header)); + const encodedPayload = base64UrlEncode(JSON.stringify(payload)); + const signature = await signHs256(params.apiSecret, `${encodedHeader}.${encodedPayload}`); + return { + token: `${encodedHeader}.${encodedPayload}.${signature}`, + expiresAt: new Date(exp * 1000).toISOString(), + }; +} + +function getVoiceSessionManifest(env: Env): { + mode: VoiceSessionMode; + transport: 'http' | 'webrtc'; + provider: VoiceSessionProvider; + ready: boolean; + planningToolsAvailableDuringConversation: boolean; + bookingToolsLockedUntilConfirm: boolean; + supportsInterruptions: boolean; + supportsServerVad: boolean; + premiumVoice: 'elevenlabs' | 'system' | 'none'; + fallback: { stt: 'whisper_proxy'; tts: 'elevenlabs_http' | 'none' }; + diagnostics: string[]; +} { + const liveProvider = getLiveVoiceProvider(env); + const diagnostics: string[] = []; + + if (env.LIVE_VOICE_RUNTIME_ENABLED !== 'true') { + diagnostics.push('live_runtime_disabled'); + } else if (!liveProvider) { + diagnostics.push('live_runtime_not_configured'); + } + + if (!env.LIVEKIT_URL || !env.LIVEKIT_API_KEY || !env.LIVEKIT_API_SECRET) { + diagnostics.push('livekit_credentials_missing'); + } + + if (!env.OPENAI_REALTIME_MODEL) { + diagnostics.push('openai_realtime_model_missing'); + } + + if (!env.ELEVENLABS_API_KEY) { + diagnostics.push('elevenlabs_unavailable'); + } + + return { + mode: liveProvider ? 'live' : 'batch', + transport: liveProvider ? 'webrtc' : 'http', + provider: liveProvider ?? 'batch_proxy', + ready: !!liveProvider, + planningToolsAvailableDuringConversation: true, + bookingToolsLockedUntilConfirm: true, + supportsInterruptions: !!liveProvider, + supportsServerVad: !!liveProvider, + premiumVoice: env.ELEVENLABS_API_KEY ? 'elevenlabs' : 'none', + fallback: { + stt: 'whisper_proxy', + tts: env.ELEVENLABS_API_KEY ? 'elevenlabs_http' : 'none', + }, + diagnostics, + }; +} + function broLog(event: string, data: Record) { console.log( JSON.stringify({ @@ -87,6 +226,76 @@ function isCfWhisperHallucination(text: string): boolean { return false; } +voiceRouter.get('/session', async (c) => { + const unauthorized = requireBroClientKey(c); + if (unauthorized) return unauthorized; + + return c.json(getVoiceSessionManifest(c.env)); +}); + +voiceRouter.post('/session/connect', async (c) => { + const unauthorized = requireBroClientKey(c); + if (unauthorized) return unauthorized; + + if (c.env.LIVE_VOICE_RUNTIME_ENABLED !== 'true') { + return c.json({ error: 'Live voice runtime is disabled' }, 503); + } + + const { LIVEKIT_URL, LIVEKIT_API_KEY, LIVEKIT_API_SECRET } = c.env; + if (!LIVEKIT_URL || !LIVEKIT_API_KEY || !LIVEKIT_API_SECRET) { + return c.json({ error: 'Live voice session is not configured' }, 503); + } + + let body: { hirerId?: string; sessionId?: string }; + try { + body = await c.req.json(); + } catch { + return c.json({ error: 'Invalid JSON body' }, 400); + } + + const hirerId = body.hirerId?.trim(); + if (!hirerId) { + return c.json({ error: 'hirerId required' }, 400); + } + + const roomName = `ace-${slugPart(hirerId, 'guest')}-${slugPart(body.sessionId ?? Date.now().toString(36), 'session')}`; + const participantIdentity = `traveler-${slugPart(hirerId, 'guest')}`; + const { token, expiresAt } = await createLiveKitParticipantToken({ + apiKey: LIVEKIT_API_KEY, + apiSecret: LIVEKIT_API_SECRET, + roomName, + participantIdentity, + metadata: { + hirerId, + sessionId: body.sessionId ?? null, + role: 'traveler', + planningToolsAvailableDuringConversation: true, + bookingToolsLockedUntilConfirm: true, + }, + }); + + const response: VoiceSessionConnectResponse = { + mode: 'live', + provider: 'livekit', + serverUrl: LIVEKIT_URL, + roomName, + participantIdentity, + participantToken: token, + expiresAt, + planningToolsAvailableDuringConversation: true, + bookingToolsLockedUntilConfirm: true, + }; + + broLog('live_session_token_issued', { + hirerId: hirerId.slice(0, 12), + roomName, + participantIdentity, + expiresAt, + }); + + return c.json(response); +}); + voiceRouter.post('/transcribe', async (c) => { const unauthorized = requireBroClientKey(c); if (unauthorized) return unauthorized; diff --git a/apps/api-edge/src/types.ts b/apps/api-edge/src/types.ts index aeab141..33614e0 100644 --- a/apps/api-edge/src/types.ts +++ b/apps/api-edge/src/types.ts @@ -214,6 +214,18 @@ export interface Env { OPENAI_API_KEY?: string; /** ElevenLabs API key — premium server-side TTS for Ace voice replies. */ ELEVENLABS_API_KEY?: string; + /** LiveKit websocket URL for persistent WebRTC voice sessions. */ + LIVEKIT_URL?: string; + /** LiveKit API key used to mint session access tokens. */ + LIVEKIT_API_KEY?: string; + /** LiveKit API secret paired with LIVEKIT_API_KEY. */ + LIVEKIT_API_SECRET?: string; + /** Explicit gate for the live voice runtime. Keep false until an Ace participant actually joins rooms and publishes audio. */ + LIVE_VOICE_RUNTIME_ENABLED?: string; + /** Optional OpenAI Realtime model name for live speech sessions. */ + OPENAI_REALTIME_MODEL?: string; + /** Optional ElevenLabs voice/agent identifier for live voice rendering. */ + ELEVENLABS_VOICE_ID?: string; /** Google Gemini API key — opt-in paid tier for high-volume extraction. Get at aistudio.google.com. Enable billing to remove RPD limits. */ GEMINI_API_KEY?: string; /** Firecrawl API key — enables markdown scraping for operators without first-party APIs. */ diff --git a/apps/meridian/app.json b/apps/meridian/app.json index 33ac7ef..50eae82 100644 --- a/apps/meridian/app.json +++ b/apps/meridian/app.json @@ -76,6 +76,7 @@ "infoPlist": { "NSMicrophoneUsageDescription": "Ace uses your microphone to capture voice requests.", "NSSpeechRecognitionUsageDescription": "Ace uses speech recognition to understand your requests.", + "NSCameraUsageDescription": "Ace uses the camera only when you add travel documents or scan booking details.", "NSFaceIDUsageDescription": "Ace uses Face ID to protect your stored credentials.", "NSLocationWhenInUseUsageDescription": "Ace uses your location to find your nearest station and suggest quick routes.", "ITSAppUsesNonExemptEncryption": false @@ -102,6 +103,8 @@ "android.permission.USE_FINGERPRINT", "android.permission.USE_BIOMETRIC", "android.permission.RECORD_AUDIO", + "android.permission.BLUETOOTH", + "android.permission.BLUETOOTH_CONNECT", "android.permission.RECEIVE_BOOT_COMPLETED", "android.permission.VIBRATE", "android.permission.ACCESS_COARSE_LOCATION", @@ -120,6 +123,18 @@ "expo-av", "expo-secure-store", "expo-local-authentication", + [ + "@livekit/react-native-expo-plugin", + { + "android": { + "audioType": "communication" + }, + "ios": { + "enableMultitaskingCameraAccess": false + } + } + ], + "@config-plugins/react-native-webrtc", [ "expo-build-properties", { diff --git a/apps/meridian/app/(main)/converse.tsx b/apps/meridian/app/(main)/converse.tsx index 12ca61a..0f59423 100644 --- a/apps/meridian/app/(main)/converse.tsx +++ b/apps/meridian/app/(main)/converse.tsx @@ -41,15 +41,17 @@ import { speakBro, cancelSpeech, preloadAudio } from '../../lib/tts'; import { appendHistory, deriveProactiveRouteMemory, loadActiveTrip, loadCurrentJourneySession, loadRouteMemories, saveJourneySession, type ActiveTrip, type RouteMemory } from '../../lib/storage'; import { planIntent, executeIntent, type ConciergePlanItem } from '../../lib/concierge'; import { shouldPreferJourney, shouldTreatTripAsLive } from '../../lib/journeyRouting'; -import { loadProfileRaw, loadProfileAuthenticated, hasProfile, type TravelProfile } from '../../lib/profile'; +import { loadProfileRaw, loadProfileAuthenticated, hasProfile, type FamilyMember, type TravelProfile } from '../../lib/profile'; import { authenticateWithBiometrics } from '../../lib/biometric'; import { getLocationContext } from '../../lib/location'; import type { StationGeo } from '../../lib/stationGeo'; import { fetchRate } from '../../lib/currency'; import { formatMoneyAmount, isZeroDecimalCurrency } from '../../lib/money'; import type { TripContext } from '../../lib/trip'; -import { rememberConfirmApproved, trackClientEvent } from '../../lib/telemetry'; +import { getLaunchSessionId, rememberConfirmApproved, trackClientEvent } from '../../lib/telemetry'; import { loadPreferredTravelUnit, type TravelUnit } from '../../lib/travelUnits'; +import { DEFAULT_VOICE_SESSION_CONFIG, isLiveVoiceSession, loadVoiceSessionConfig } from '../../lib/voiceSession'; +import { connectLiveVoiceSession, type LiveVoiceConnection } from '../../lib/liveVoice'; type MarketNationality = 'uk' | 'india' | 'other'; @@ -101,6 +103,17 @@ type FollowUpContext = { plan: ConciergePlanItem[]; }; +type PassengerIntentTraveller = { + name: string; + relationship: 'adult' | 'child' | 'infant'; +}; + +type PassengerIntent = { + mode: BookingMode; + travellers: PassengerIntentTraveller[]; + summaryLine: string | null; +}; + function looksLikeOptionSelection(text: string): boolean { const normalized = text.trim().toLowerCase(); if (!normalized) return false; @@ -423,6 +436,136 @@ function referencesSharedTravel(text: string): boolean { ].some((pattern) => pattern.test(text)); } +function escapeRegExp(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function firstName(name: string): string { + return name.trim().split(/\s+/)[0] ?? name.trim(); +} + +function nameAppearsInText(text: string, name: string): boolean { + const trimmed = name.trim(); + if (!trimmed) return false; + const normalizedText = ` ${text.toLowerCase()} `; + const fullNamePattern = new RegExp(`(^|[^a-z])${escapeRegExp(trimmed.toLowerCase())}([^a-z]|$)`, 'i'); + if (fullNamePattern.test(normalizedText)) return true; + + const givenName = firstName(trimmed); + if (givenName.length < 3 || givenName.toLowerCase() === 'you') return false; + const givenNamePattern = new RegExp(`(^|[^a-z])${escapeRegExp(givenName.toLowerCase())}([^a-z]|$)`, 'i'); + return givenNamePattern.test(normalizedText); +} + +function familyMemberToTraveller(member: FamilyMember): PassengerIntentTraveller { + return { + name: member.name.trim(), + relationship: member.relationship, + }; +} + +function travelUnitMemberToTraveller(member: TravelUnit['members'][number]): PassengerIntentTraveller | null { + if (member.role === 'self') { + return { + name: 'You', + relationship: 'adult', + }; + } + + const name = member.name.trim(); + if (!name) return null; + return { + name, + relationship: member.role === 'child' || member.role === 'infant' ? member.role : 'adult', + }; +} + +function dedupeTravellers(travellers: PassengerIntentTraveller[]): PassengerIntentTraveller[] { + const seen = new Set(); + const unique: PassengerIntentTraveller[] = []; + for (const traveller of travellers) { + const key = `${traveller.name.trim().toLowerCase()}::${traveller.relationship}`; + if (!traveller.name.trim() || seen.has(key)) continue; + seen.add(key); + unique.push(traveller); + } + return unique; +} + +function describeTravellers(travellers: PassengerIntentTraveller[]): string { + if (travellers.length === 0) return 'you'; + const labels = travellers.map((traveller) => traveller.name === 'You' ? 'you' : traveller.name); + if (labels.length === 1) return labels[0]; + if (labels.length === 2) return `${labels[0]} and ${labels[1]}`; + return `${labels.slice(0, -1).join(', ')}, and ${labels.at(-1)}`; +} + +function buildPassengerIntentPlanningNote(intent: PassengerIntent): string { + if (intent.mode !== 'shared' || intent.travellers.length <= 1 || !intent.summaryLine) { + return ` + +Ace traveller context: +- This request is for the device owner only. +- Do not assume saved companions or family members are travelling unless the customer names them or asks for a shared trip.`; + } + return ` + +Ace traveller context: +- ${intent.summaryLine} +- Treat pricing, availability, and booking scope as a ${intent.travellers.length}-traveller request for ${describeTravellers(intent.travellers)}.`; +} + +function inferPassengerIntent(params: { + text: string; + bookingMode: BookingMode; + preferredTravelUnit: TravelUnit | null; + familyMembers: FamilyMember[]; +}): PassengerIntent { + const familyTravellers = params.familyMembers + .filter((member) => member.name.trim()) + .map(familyMemberToTraveller); + const sharedUnitTravellers = params.preferredTravelUnit?.members + .map(travelUnitMemberToTraveller) + .filter((traveller): traveller is PassengerIntentTraveller => !!traveller) ?? []; + const matchedFamilyTravellers = familyTravellers.filter((traveller) => nameAppearsInText(params.text, traveller.name)); + const matchedSharedTravellers = sharedUnitTravellers.filter((traveller) => traveller.name !== 'You' && nameAppearsInText(params.text, traveller.name)); + const namedTravellers = dedupeTravellers([{ name: 'You', relationship: 'adult' }, ...matchedSharedTravellers, ...matchedFamilyTravellers]); + + if (namedTravellers.length > 1) { + return { + mode: 'shared', + travellers: namedTravellers, + summaryLine: `The customer explicitly asked Ace to book for ${describeTravellers(namedTravellers)}.`, + }; + } + + if (params.preferredTravelUnit && (params.bookingMode === 'shared' || referencesSharedTravel(params.text))) { + const travellers = dedupeTravellers(sharedUnitTravellers); + if (travellers.length > 1) { + return { + mode: 'shared', + travellers, + summaryLine: `Use the saved shared travel unit ${params.preferredTravelUnit.name} for this booking.`, + }; + } + } + + if (referencesSharedTravel(params.text) && familyTravellers.length > 0) { + const travellers = dedupeTravellers([{ name: 'You', relationship: 'adult' }, ...familyTravellers]); + return { + mode: 'shared', + travellers, + summaryLine: `The customer asked Ace to book shared travel for ${describeTravellers(travellers)}.`, + }; + } + + return { + mode: 'solo', + travellers: [{ name: 'You', relationship: 'adult' }], + summaryLine: null, + }; +} + function buildSharedTravelUnitContext(unit: TravelUnit): Record { return { id: unit.id, @@ -493,6 +636,19 @@ function buildSharedTravelReadiness(unit: TravelUnit | null): SharedTravelReadin }; } +function buildPassengerDisplay( + passengers: Array<{ name: string; relationship: string }> | null, + fallback: string, +): string { + if (!passengers || passengers.length === 0) return fallback; + const names = passengers + .map((passenger) => passenger.name?.trim()) + .filter(Boolean); + if (names.length === 0) return fallback; + if (names.length <= 3) return names.join(', '); + return `${names.slice(0, 2).join(', ')} +${names.length - 2} more`; +} + function getCountdown(departureTime: string | null | undefined): string | null { if (!departureTime) return null; const d = new Date(departureTime); @@ -765,10 +921,24 @@ export default function ConverseScreen() { const [nearestStation, setNearestStation] = useState(null); const [locationLabel, setLocationLabel] = useState(null); const [voiceEnabled, setVoiceEnabled] = useState(true); + const [voiceSessionConfig, setVoiceSessionConfig] = useState(DEFAULT_VOICE_SESSION_CONFIG); const [isSpeaking, setIsSpeaking] = useState(false); + const [liveVoiceConnected, setLiveVoiceConnected] = useState(false); + const [liveVoiceConnecting, setLiveVoiceConnecting] = useState(false); + const [liveVoiceMicEnabled, setLiveVoiceMicEnabled] = useState(false); + const [liveRemoteSpeaking, setLiveRemoteSpeaking] = useState(false); + const [liveCaption, setLiveCaption] = useState(null); + const [liveHeardLine, setLiveHeardLine] = useState(null); const micAmplitude = useSharedValue(0); const ttsAmplitude = useSharedValue(0); + const liveVoiceConnectionRef = useRef(null); + const liveVoiceConnectAttemptedRef = useRef(false); + const liveTranscriptBufferRef = useRef(''); + const liveRemotePresenceSeenRef = useRef(false); + const liveRemoteMessageIdsRef = useRef(new Set()); + const liveVoicePresenceTimerRef = useRef | null>(null); const beginVoiceCaptureRef = useRef<(() => Promise) | null>(null); + const speakIfEnabledRef = useRef<((text: string, restartListening?: boolean) => Promise) | null>(null); const handsFreeListenTimerRef = useRef | null>(null); const nextSilenceMsRef = useRef(2800); const keyboardVisibleRef = useRef(false); @@ -802,6 +972,228 @@ export default function ConverseScreen() { }); }, []); + const clearLiveVoicePresenceTimer = useCallback(() => { + if (liveVoicePresenceTimerRef.current) { + clearTimeout(liveVoicePresenceTimerRef.current); + liveVoicePresenceTimerRef.current = null; + } + }, []); + + const surfaceLiveVoiceFallback = useCallback((message?: string) => { + const nextMessage = message?.trim() || 'Live voice is unavailable right now. Type the trip below and Ace will keep moving.'; + clearLiveVoicePresenceTimer(); + liveVoiceConnectionRef.current = null; + liveTranscriptBufferRef.current = ''; + liveRemotePresenceSeenRef.current = false; + setLiveVoiceConnected(false); + setLiveVoiceConnecting(false); + setLiveVoiceMicEnabled(false); + setLiveRemoteSpeaking(false); + setLiveCaption(null); + setLiveHeardLine(null); + setVoiceSessionConfig(DEFAULT_VOICE_SESSION_CONFIG); + phaseRef.current = 'error'; + setPhase('error'); + setError(nextMessage); + setTextFallbackVisible(true); + }, [clearLiveVoicePresenceTimer, setError, setPhase]); + + const degradeLiveVoiceToBatch = useCallback(async () => { + clearLiveVoicePresenceTimer(); + const current = liveVoiceConnectionRef.current; + liveVoiceConnectionRef.current = null; + liveTranscriptBufferRef.current = ''; + liveRemotePresenceSeenRef.current = false; + setLiveVoiceConnected(false); + setLiveVoiceConnecting(false); + setLiveVoiceMicEnabled(false); + setLiveRemoteSpeaking(false); + setLiveCaption(null); + setLiveHeardLine(null); + setVoiceSessionConfig(DEFAULT_VOICE_SESSION_CONFIG); + setError(null); + setTextFallbackVisible(false); + phaseRef.current = 'idle'; + setPhase('idle'); + + if (current) { + await current.disconnect().catch(() => {}); + } + + const hour = new Date().getHours(); + const openingLine = hour < 12 ? 'Good morning.' : hour < 17 ? 'Good afternoon.' : 'Good evening.'; + await speakIfEnabledRef.current?.(openingLine, true); + }, [clearLiveVoicePresenceTimer, setError, setPhase]); + + const disconnectLiveVoice = useCallback(async () => { + const current = liveVoiceConnectionRef.current; + clearLiveVoicePresenceTimer(); + liveVoiceConnectionRef.current = null; + liveTranscriptBufferRef.current = ''; + liveRemotePresenceSeenRef.current = false; + setLiveVoiceConnected(false); + setLiveVoiceConnecting(false); + setLiveVoiceMicEnabled(false); + setLiveRemoteSpeaking(false); + setLiveCaption(null); + setLiveHeardLine(null); + if (current) { + await current.disconnect().catch(() => {}); + } + }, [clearLiveVoicePresenceTimer]); + + const connectLiveVoice = useCallback(async () => { + if (!agentId || !isLiveVoiceSession(voiceSessionConfig)) return; + if (liveVoiceConnectionRef.current || liveVoiceConnecting) return; + + clearLiveVoicePresenceTimer(); + liveRemotePresenceSeenRef.current = false; + setLiveVoiceConnecting(true); + try { + const connection = await connectLiveVoiceSession({ + hirerId: agentId, + sessionId: getLaunchSessionId(), + callbacks: { + onConnected: () => { + setLiveVoiceConnected(true); + setLiveVoiceConnecting(false); + setLiveVoiceMicEnabled(true); + setLiveCaption(null); + phaseRef.current = 'listening'; + setPhase('listening'); + logConverseEvent({ + event: 'live_voice_connected', + metadata: { + provider: voiceSessionConfig.provider, + transport: voiceSessionConfig.transport, + }, + }); + clearLiveVoicePresenceTimer(); + liveVoicePresenceTimerRef.current = setTimeout(() => { + if (liveRemotePresenceSeenRef.current) return; + logConverseEvent({ + event: 'live_voice_room_empty', + severity: 'warning', + message: 'Live room connected without an Ace participant. Falling back to batch voice.', + }); + void degradeLiveVoiceToBatch(); + }, 2500); + }, + onDisconnected: () => { + liveVoiceConnectionRef.current = null; + liveTranscriptBufferRef.current = ''; + clearLiveVoicePresenceTimer(); + liveRemotePresenceSeenRef.current = false; + setLiveVoiceConnected(false); + setLiveVoiceMicEnabled(false); + setLiveRemoteSpeaking(false); + setLiveCaption(null); + if (phaseRef.current === 'listening') { + phaseRef.current = 'idle'; + setPhase('idle'); + } + logConverseEvent({ event: 'live_voice_disconnected' }); + }, + onReconnecting: () => { + setLiveVoiceConnecting(true); + logConverseEvent({ event: 'live_voice_reconnecting' }); + }, + onReconnected: () => { + setLiveVoiceConnected(true); + setLiveVoiceConnecting(false); + clearLiveVoicePresenceTimer(); + logConverseEvent({ event: 'live_voice_reconnected' }); + }, + onRemoteSpeakingChanged: (speaking) => { + if (speaking) { + liveRemotePresenceSeenRef.current = true; + clearLiveVoicePresenceTimer(); + } + setLiveRemoteSpeaking(speaking); + }, + onChatMessage: async (message, participant) => { + const localParticipantIdentity = liveVoiceConnectionRef.current?.participantIdentity; + if (!participant || participant.identity === localParticipantIdentity) return; + if (liveRemoteMessageIdsRef.current.has(message.id)) return; + liveRemoteMessageIdsRef.current.add(message.id); + liveRemotePresenceSeenRef.current = true; + clearLiveVoicePresenceTimer(); + const narration = sanitizeAceNarration(message.message); + if (!narration) return; + setLiveCaption(narration); + const meridianTurn = { role: 'meridian' as const, text: narration, ts: Date.now() }; + addTurn(meridianTurn); + await appendHistory(meridianTurn).catch(() => {}); + logConverseEvent({ + event: 'live_voice_message_received', + metadata: { + participantIdentity: participant.identity, + textLength: narration.length, + }, + }); + }, + onTranscriptionReceived: (segments, participant) => { + const localParticipantIdentity = liveVoiceConnectionRef.current?.participantIdentity; + const joined = segments + .map((segment) => segment.text.trim()) + .filter(Boolean) + .join(' ') + .trim(); + if (!joined) return; + + if (participant?.identity === localParticipantIdentity) { + liveTranscriptBufferRef.current = joined; + setLiveHeardLine(joined); + logConverseEvent({ + event: 'live_voice_local_transcript', + metadata: { + final: segments.every((segment) => segment.final), + textLength: joined.length, + }, + }); + return; + } + + liveRemotePresenceSeenRef.current = true; + clearLiveVoicePresenceTimer(); + setLiveCaption(joined); + logConverseEvent({ + event: 'live_voice_remote_transcript', + metadata: { + final: segments.every((segment) => segment.final), + textLength: joined.length, + participantIdentity: participant?.identity ?? null, + }, + }); + }, + onError: (error) => { + liveVoiceConnectionRef.current = null; + const message = 'Live voice is unavailable right now. Type the trip below and Ace will keep moving.'; + surfaceLiveVoiceFallback(message); + logConverseEvent({ + event: 'live_voice_error', + severity: 'warning', + message: error.message || message, + }); + }, + }, + }); + + liveVoiceConnectionRef.current = connection; + } catch (error: any) { + liveVoiceConnectionRef.current = null; + clearLiveVoicePresenceTimer(); + liveRemotePresenceSeenRef.current = false; + const message = 'Ace could not open live voice just now. Type the trip below and it will still handle the route.'; + surfaceLiveVoiceFallback(message); + logConverseEvent({ + event: 'live_voice_connect_failed', + severity: 'warning', + message: error?.message ?? message, + }); + } + }, [addTurn, agentId, clearLiveVoicePresenceTimer, degradeLiveVoiceToBatch, liveVoiceConnecting, logConverseEvent, setPhase, surfaceLiveVoiceFallback, voiceSessionConfig]); + useEffect(() => { if (turns.length === 0) return; setTimeout(() => scrollRef.current?.scrollToEnd({ animated: true }), 100); @@ -1006,6 +1398,26 @@ export default function ConverseScreen() { }, 80); }, [activeTrip, clearHandsFreeListenTimer, setPhase, voiceEnabled]); + const openTextFallback = useCallback((message: string, seed?: string) => { + clearHandsFreeListenTimer(); + cancelSpeech(); + setIsSpeaking(false); + ttsAmplitude.value = 0; + if (phaseRef.current === 'listening') { + void stopRecording().catch(() => null); + recordingActiveRef.current = false; + startingRecordingRef.current = false; + micAmplitude.value = 0; + phaseRef.current = 'idle'; + setPhase('idle'); + } + setError(message); + setTextFallbackVisible(true); + if (seed != null) { + setTextFallbackDraft(seed); + } + }, [clearHandsFreeListenTimer, setError, setPhase]); + const speakIfEnabled = useCallback(async (text: string, restartListening = true) => { if (!voiceEnabled) return; clearHandsFreeListenTimer(); @@ -1023,6 +1435,12 @@ export default function ConverseScreen() { setIsSpeaking(false); ttsAmplitude.value = 0; } + if (!speakingStarted) { + openTextFallback( + 'Ace voice is unavailable right now. Type the trip once below and Ace will keep the journey moving.', + ); + return; + } if (restartListening && !keyboardVisibleRef.current && !textFallbackVisibleRef.current) { const cur = phaseRef.current; // Don't auto-restart when a booking card is showing, payment is processing, or booking is done @@ -1030,27 +1448,7 @@ export default function ConverseScreen() { armHandsFreeListen(1600); } } - }, [armHandsFreeListen, clearHandsFreeListenTimer, voiceEnabled]); - - const openTextFallback = useCallback((message: string, seed?: string) => { - clearHandsFreeListenTimer(); - cancelSpeech(); - setIsSpeaking(false); - ttsAmplitude.value = 0; - if (phaseRef.current === 'listening') { - void stopRecording().catch(() => null); - recordingActiveRef.current = false; - startingRecordingRef.current = false; - micAmplitude.value = 0; - phaseRef.current = 'idle'; - setPhase('idle'); - } - setError(message); - setTextFallbackVisible(true); - if (seed != null) { - setTextFallbackDraft(seed); - } - }, [clearHandsFreeListenTimer, setError, setPhase]); + }, [armHandsFreeListen, clearHandsFreeListenTimer, openTextFallback, voiceEnabled]); // ── Phase 1: plan ───────────────────────────────────────────────────────── @@ -1062,12 +1460,10 @@ export default function ConverseScreen() { const followUpContext = shouldUseFollowUpContext(displayText, followUpContextRef.current) ? followUpContextRef.current : null; - const planningTranscript = buildFollowUpPlanningTranscript({ + const basePlanningTranscript = buildFollowUpPlanningTranscript({ reply: prepared.planningTranscript, context: followUpContext, }); - const usingSharedTravel = - !!preferredTravelUnit && (bookingMode === 'shared' || referencesSharedTravel(displayText)); setTextFallbackVisible(false); setTextFallbackDraft(''); @@ -1075,25 +1471,27 @@ export default function ConverseScreen() { setTranscript(displayText); setPhase('thinking'); const planStartedAt = Date.now(); - logConverseEvent({ - event: 'plan_requested', - metadata: { - bookingMode: usingSharedTravel ? 'shared' : 'solo', - textLength: displayText.length, - usedFollowUpContext: !!followUpContext, - usedLocationAssumption: !!prepared.assumptionNote, - }, - }); // Load travel profile for Phase 1 — preferences only, no identity data. // Full profile (legalName, email, phone, documents) is loaded AFTER biometric // confirmation in handleBiometricConfirm. This ensures sensitive data never // reaches memory before the user explicitly authorises the booking. let profile: TravelProfile | null = null; + let passengerIntent: PassengerIntent = { + mode: 'solo', + travellers: [{ name: 'You', relationship: 'adult' }], + summaryLine: null, + }; let travelProfile: Record | undefined; try { if (await hasProfile()) { profile = await loadProfileRaw(); + passengerIntent = inferPassengerIntent({ + text: displayText, + bookingMode, + preferredTravelUnit, + familyMembers: profile?.familyMembers ?? [], + }); if (profile) { // Phase 1: non-identity prefs only — railcard/class/nationality for narration personalisation. // familyMembers included (no documents or contact details) so Claude can resolve names. @@ -1112,8 +1510,10 @@ export default function ConverseScreen() { railcard: m.railcard, nationality: m.nationality, })), + requestedTravellers: passengerIntent.travellers, + requestedTravelMode: passengerIntent.mode, }; - if (usingSharedTravel && preferredTravelUnit) { + if (passengerIntent.mode === 'shared' && preferredTravelUnit) { travelProfile.sharedTravelUnit = buildSharedTravelUnitContext(preferredTravelUnit); } } @@ -1122,9 +1522,39 @@ export default function ConverseScreen() { // No profile stored — proceed without } + if (!profile) { + passengerIntent = inferPassengerIntent({ + text: displayText, + bookingMode, + preferredTravelUnit, + familyMembers: [], + }); + } + + const planningTranscript = `${basePlanningTranscript}${buildPassengerIntentPlanningNote(passengerIntent)}`; + const usingSharedTravel = passengerIntent.mode === 'shared' && passengerIntent.travellers.length > 1; + + logConverseEvent({ + event: 'plan_requested', + metadata: { + bookingMode: usingSharedTravel ? 'shared' : 'solo', + textLength: displayText.length, + usedFollowUpContext: !!followUpContext, + usedLocationAssumption: !!prepared.assumptionNote, + travellerCount: passengerIntent.travellers.length, + }, + }); + if (!travelProfile && usingSharedTravel && preferredTravelUnit) { travelProfile = { sharedTravelUnit: buildSharedTravelUnitContext(preferredTravelUnit), + requestedTravellers: passengerIntent.travellers, + requestedTravelMode: passengerIntent.mode, + }; + } else if (!travelProfile) { + travelProfile = { + requestedTravellers: passengerIntent.travellers, + requestedTravelMode: passengerIntent.mode, }; } @@ -1261,12 +1691,59 @@ export default function ConverseScreen() { // Pre-fetch the opening greeting the moment the screen mounts so the audio // file is ready before the 1.2s timer fires — instant first sound. useEffect(() => { - if (!voiceEnabled) return; + let cancelled = false; + + void loadVoiceSessionConfig() + .then((config) => { + if (cancelled) return; + setVoiceSessionConfig(config); + void trackClientEvent({ + event: 'voice_runtime_configured', + screen: 'converse', + metadata: { + mode: config.mode, + transport: config.transport, + provider: config.provider, + ready: config.ready, + supportsInterruptions: config.supportsInterruptions, + supportsServerVad: config.supportsServerVad, + premiumVoice: config.premiumVoice, + planningToolsAvailableDuringConversation: config.planningToolsAvailableDuringConversation, + bookingToolsLockedUntilConfirm: config.bookingToolsLockedUntilConfirm, + diagnostics: config.diagnostics, + }, + }); + }) + .catch(() => null); + + return () => { + cancelled = true; + }; + }, []); + + useEffect(() => { + if (!agentId || !voiceEnabled || !isLiveVoiceSession(voiceSessionConfig)) return; + if (liveVoiceConnectAttemptedRef.current) return; + liveVoiceConnectAttemptedRef.current = true; + void connectLiveVoice(); + }, [agentId, connectLiveVoice, voiceEnabled, voiceSessionConfig]); + + useEffect(() => { + if (voiceEnabled) return; + void disconnectLiveVoice(); + }, [disconnectLiveVoice, voiceEnabled]); + + useEffect(() => () => { + void disconnectLiveVoice(); + }, [disconnectLiveVoice]); + + useEffect(() => { + if (!voiceEnabled || isLiveVoiceSession(voiceSessionConfig)) return; const h = new Date().getHours(); const line = h < 12 ? 'Good morning.' : h < 17 ? 'Good afternoon.' : 'Good evening.'; void preloadAudio(line); // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); + }, [voiceEnabled, voiceSessionConfig]); // Always-on: auto-start listening 1.2s after mount once agentId is ready. // Skipped if a prefill is pending (it will take priority) or a trip is active. @@ -1280,6 +1757,10 @@ export default function ConverseScreen() { if (liveSession && shouldPreferJourney(liveSession)) return; // Only start if we're still idle and the keyboard isn't up if (phaseRef.current === 'idle' && !keyboardVisibleRef.current && !textFallbackVisibleRef.current) { + if (isLiveVoiceSession(voiceSessionConfig)) { + void connectLiveVoice(); + return; + } // Speak a brief greeting so the user knows Ace is ready, then // speakIfEnabled's restartListening path automatically arms the mic. const h = new Date().getHours(); @@ -1289,10 +1770,10 @@ export default function ConverseScreen() { })(); }, 1200); return () => clearTimeout(timer); - }, [activeTrip, agentId, prefill, voiceEnabled]); + }, [activeTrip, agentId, connectLiveVoice, prefill, voiceEnabled, voiceSessionConfig]); useEffect(() => { - if (phase !== 'done' || isSpeaking) return; + if (phase !== 'done' || isSpeaking || liveRemoteSpeaking) return; const timer = setTimeout(() => { if (phaseRef.current !== 'done') return; setPhase('idle'); @@ -1307,7 +1788,7 @@ export default function ConverseScreen() { } }, 4000); return () => clearTimeout(timer); - }, [activeTrip, isSpeaking, phase, setPhase, voiceEnabled]); + }, [activeTrip, isSpeaking, liveRemoteSpeaking, phase, setPhase, voiceEnabled]); const handleTextFallbackSend = useCallback(async () => { const text = textFallbackDraft.trim(); @@ -1536,7 +2017,14 @@ export default function ConverseScreen() { // This is the first time legalName, email, phone, and documents enter memory. try { const fullProfile = await loadProfileAuthenticated(); - if (fullProfile) pending.fullProfile = fullProfile as unknown as Record; + if (fullProfile) { + pending.fullProfile = { + ...(fullProfile as unknown as Record), + requestedTravellers: pending.travelProfile?.requestedTravellers, + requestedTravelMode: pending.travelProfile?.requestedTravelMode, + sharedTravelUnit: pending.travelProfile?.sharedTravelUnit, + }; + } } catch { // Biometric already succeeded — proceed without full profile if load fails } @@ -1574,6 +2062,31 @@ export default function ConverseScreen() { const finishingRecordingRef = useRef(false); const finishVoiceCapture = useCallback(async () => { + if (isLiveVoiceSession(voiceSessionConfig)) { + clearHandsFreeListenTimer(); + if (liveVoiceConnectionRef.current) { + await liveVoiceConnectionRef.current.setMicrophoneEnabled(false).catch(() => {}); + } + if (liveTranscriptBufferRef.current.trim()) { + const heard = liveTranscriptBufferRef.current.trim(); + setTranscript(heard); + addTurn({ role: 'user', text: heard, ts: Date.now() }); + await appendHistory({ role: 'user', text: heard, ts: Date.now() }).catch(() => {}); + logConverseEvent({ + event: 'live_voice_local_turn_captured', + metadata: { textLength: heard.length }, + }); + } + liveTranscriptBufferRef.current = ''; + setLiveVoiceMicEnabled(false); + setLiveRemoteSpeaking(false); + setLiveHeardLine(null); + micAmplitude.value = 0; + phaseRef.current = 'idle'; + setPhase('idle'); + return; + } + if (finishingRecordingRef.current) return; clearHandsFreeListenTimer(); if (startingRecordingRef.current && !recordingActiveRef.current) { @@ -1628,10 +2141,10 @@ export default function ConverseScreen() { }); let nextMessage = __DEV__ ? `STT: ${rawMessage}` : 'Voice error — try again.'; if (!__DEV__) { - nextMessage = 'Ace missed that. Say it again, or type the trip below.'; + nextMessage = 'I did not catch the trip clearly. Say it once more, or type it below.'; } if (msg.includes('timed out') || msg.includes('timeout') || msg.includes('abort')) { - nextMessage = 'Ace did not catch that in time. Say it again, or type it below.'; + nextMessage = 'I did not catch that in time. Say it once more, or type it below.'; } else if (msg.includes('401') || msg.includes('403') || msg.includes('not authorised')) { nextMessage = 'Voice service is unavailable right now. You can type the trip below.'; } else if (msg.includes('503') || msg.includes('not configured')) { @@ -1705,7 +2218,7 @@ export default function ConverseScreen() { message: rawMessage, metadata: { phase: 'capture' }, }); - let nextMessage = 'Ace missed that. Say it again, or type the trip below.'; + let nextMessage = 'I did not catch the trip clearly. Say it once more, or type it below.'; if (msg.includes('timed out') || msg.includes('timeout')) { nextMessage = 'Ace took too long on that request. Try again, or type it below.'; } else if (msg.includes('no connection') || msg.includes('internet') || msg.includes('network')) { @@ -1725,10 +2238,45 @@ export default function ConverseScreen() { startingRecordingRef.current = false; finishingRecordingRef.current = false; } - }, [clearHandsFreeListenTimer, logConverseEvent, openTextFallback, runIntentWithUiFallback, setError, setPhase]); + }, [addTurn, clearHandsFreeListenTimer, logConverseEvent, openTextFallback, runIntentWithUiFallback, setError, setPhase, setTranscript, voiceSessionConfig]); const beginVoiceCapture = useCallback(async () => { if (!voiceEnabled) return; + if (isLiveVoiceSession(voiceSessionConfig)) { + const currentPhase = phaseRef.current; + clearHandsFreeListenTimer(); + cancelSpeech(); + if (currentPhase === 'error') reset(); + setTextFallbackVisible(false); + setError(null); + setLiveHeardLine(null); + liveTranscriptBufferRef.current = ''; + voiceCaptureStartedAtRef.current = Date.now(); + logConverseEvent({ + event: 'voice_capture_started', + metadata: { fromPhase: currentPhase, live: true }, + }); + logConverseEvent({ + event: 'voice_session_start', + metadata: { + fromPhase: currentPhase, + voiceMode: voiceSessionConfig.mode, + voiceTransport: voiceSessionConfig.transport, + voiceProvider: voiceSessionConfig.provider, + }, + }); + + if (!liveVoiceConnectionRef.current) { + await connectLiveVoice(); + } else { + await liveVoiceConnectionRef.current.setMicrophoneEnabled(true).catch(() => {}); + setLiveVoiceMicEnabled(true); + phaseRef.current = 'listening'; + setPhase('listening'); + } + return; + } + if (recordingActiveRef.current || startingRecordingRef.current || finishingRecordingRef.current) return; const currentPhase = phaseRef.current; if (currentPhase !== 'idle' && currentPhase !== 'error') return; @@ -1746,7 +2294,12 @@ export default function ConverseScreen() { }); logConverseEvent({ event: 'voice_session_start', - metadata: { fromPhase: currentPhase }, + metadata: { + fromPhase: currentPhase, + voiceMode: voiceSessionConfig.mode, + voiceTransport: voiceSessionConfig.transport, + voiceProvider: voiceSessionConfig.provider, + }, }); startingRecordingRef.current = true; try { @@ -1784,17 +2337,29 @@ export default function ConverseScreen() { message, }); } - }, [clearHandsFreeListenTimer, finishVoiceCapture, logConverseEvent, reset, setError, setPhase, voiceEnabled]); + }, [clearHandsFreeListenTimer, connectLiveVoice, finishVoiceCapture, logConverseEvent, reset, setError, setPhase, voiceEnabled, voiceSessionConfig]); // Wire beginVoiceCapture into ref so speakIfEnabled can auto-restart without circular dep beginVoiceCaptureRef.current = beginVoiceCapture; // Wire speakIfEnabled into ref so the always-on greeting can use the latest closure - const speakIfEnabledRef = useRef<((text: string, restartListening?: boolean) => Promise) | null>(null); speakIfEnabledRef.current = speakIfEnabled; const handleOrbTap = useCallback(async () => { clearHandsFreeListenTimer(); + if (isLiveVoiceSession(voiceSessionConfig)) { + if (liveVoiceConnecting) return; + if (!liveVoiceConnectionRef.current) { + await beginVoiceCapture(); + return; + } + if (liveVoiceMicEnabled) { + await finishVoiceCapture(); + return; + } + await beginVoiceCapture(); + return; + } if (phase === 'listening') { await finishVoiceCapture(); return; @@ -1809,7 +2374,7 @@ export default function ConverseScreen() { } nextSilenceMsRef.current = 2800; await beginVoiceCapture(); - }, [beginVoiceCapture, clearHandsFreeListenTimer, finishVoiceCapture, phase]); + }, [beginVoiceCapture, clearHandsFreeListenTimer, finishVoiceCapture, liveVoiceConnecting, liveVoiceMicEnabled, phase, voiceSessionConfig]); const handleShortcutIntent = useCallback(async (destination: string, kind: 'home' | 'work') => { if (!nearestStation) return; @@ -1853,22 +2418,25 @@ export default function ConverseScreen() { const isIdle = phase === 'idle'; const isError = phase === 'error'; const isConfirming = phase === 'confirming'; + const liveVoiceReady = isLiveVoiceSession(voiceSessionConfig); const presenceLabel = - isSpeaking ? 'Ace is speaking' : - phase === 'listening' ? 'Ace is listening' : + isSpeaking || liveRemoteSpeaking ? 'Ace is speaking' : + liveVoiceConnecting ? 'Ace is joining' : + phase === 'listening' ? (liveVoiceReady ? 'Ace is with you' : 'Ace is listening') : phase === 'thinking' ? 'Ace is thinking' : phase === 'hiring' || phase === 'executing' ? 'Ace is securing your trip' : phase === 'done' ? 'Trip secured' : phase === 'error' ? 'Ace is waiting' : - voiceEnabled ? 'Ace is ready' : + voiceEnabled ? (liveVoiceReady ? 'Ace is live' : 'Ace is ready') : 'Voice paused'; const presenceHint = - isSpeaking ? 'Stay with me. Ace is guiding the next move.' : - phase === 'listening' ? 'Say the trip once, naturally.' : + isSpeaking || liveRemoteSpeaking ? 'Stay with me. Ace is guiding the next move.' : + liveVoiceConnecting ? 'Joining your live concierge session.' : + phase === 'listening' ? (liveVoiceReady ? 'Speak naturally. Ace will stay with the thread.' : 'Say the trip once, naturally.') : phase === 'thinking' || phase === 'hiring' || phase === 'executing' ? 'Ace is working the route, timing, and booking.' : phase === 'done' ? 'Trip secured. Ace will stay with it.' : - phase === 'error' ? 'Say it again or continue in text below.' : - voiceEnabled ? 'Say the trip when you are ready.' : + phase === 'error' ? 'Say it once more, or continue in text below.' : + voiceEnabled ? (liveVoiceReady ? 'Ace is live. Say the trip when you are ready.' : 'Say the trip when you are ready.') : 'Tap Ace to resume voice.'; const presenceTone = phase === 'listening' ? '#c8e8ff' : @@ -1907,6 +2475,10 @@ export default function ConverseScreen() { const heroResponse = bookingMode === 'shared' ? `I'll line this up for ${preferredTravelUnit?.name?.toLowerCase() ?? 'both of you'}.` + : !preferredTravelUnit && familyMemberCount >= 2 + ? 'I can move just for you now, or set up family mode once so “for the family” becomes natural.' + : !preferredTravelUnit && familyMemberCount === 1 + ? 'I can move just for you now, or set up couple mode once so “for us” becomes natural.' : routeMemory ? `I can line up ${routeMemory.origin} to ${routeMemory.destination} again, or take you somewhere new.` : nearestStation @@ -1922,16 +2494,15 @@ export default function ConverseScreen() { ? `Ace has seen this route ${routeMemory.count} times and can line it up before you ask tomorrow.` : `Ace remembers this route and can line it up quickly when you need it.` : null; - const shouldShowTravelModeReminder = guidanceSessions >= 3 && guidanceSessions % 6 === 0; - const shouldNudgeTravelSetup = - !preferredTravelUnit - && !travelModePromptDismissed - && guidanceSessions >= 2 - && guidanceSessions % 4 === 0; const inferredTravelShape = familyMemberCount >= 2 ? 'family' : familyMemberCount === 1 ? 'couple' : 'single'; + const shouldShowTravelModeReminder = guidanceSessions >= 3 && guidanceSessions % 6 === 0; + const shouldNudgeTravelSetup = + !preferredTravelUnit + && !travelModePromptDismissed + && inferredTravelShape !== 'single'; const travelModeReminder = preferredTravelUnit ? shouldShowTravelModeReminder ? { @@ -1978,6 +2549,12 @@ export default function ConverseScreen() { // Last route suggestion — "Same route as last time?" const recentTurns = turns.filter((turn) => turn.role === 'meridian').slice(-4); + const voiceFirstIdle = turns.length === 0 && voiceEnabled && !textFallbackVisible && (phase === 'idle' || phase === 'listening'); + const showActiveTripCard = !!activeTrip; + const showUsualRouteCard = !activeTrip && !!usualRoute; + const showMemoryCard = !activeTrip && !usualRoute && !!routeMemory && !!memorySuggestion; + const showSharedUnitCard = !activeTrip && !usualRoute && !routeMemory && !!preferredTravelUnit; + const showTravelModeCard = !activeTrip && !usualRoute && !routeMemory && !preferredTravelUnit && !!travelModeReminder; return ( @@ -2062,7 +2639,7 @@ export default function ConverseScreen() { )} - {activeTrip && ( + {showActiveTripCard && ( { router.push({ @@ -2101,7 +2678,7 @@ export default function ConverseScreen() { })()} )} - {usualRoute && ( + {showUsualRouteCard && ( { void runIntentWithUiFallback(`${usualRoute.origin} to ${usualRoute.destination}`); }} style={styles.usualRouteCard} @@ -2117,7 +2694,7 @@ export default function ConverseScreen() { )} )} - {!usualRoute && routeMemory && memorySuggestion && ( + {showMemoryCard && ( { void runIntentWithUiFallback(memorySuggestion); }} style={styles.usualRouteCard} @@ -2133,7 +2710,7 @@ export default function ConverseScreen() { )} )} - {preferredTravelUnit && ( + {showSharedUnitCard && ( @@ -2161,7 +2738,7 @@ export default function ConverseScreen() { )} - {travelModeReminder && ( + {travelModeReminder && (!voiceFirstIdle || showTravelModeCard) && ( {travelModeReminder.eyebrow} @@ -2186,12 +2763,14 @@ export default function ConverseScreen() { )} + {!voiceFirstIdle && ( router.push('/(main)/trips')} style={styles.myTripsBtn}> Journeys - {primarySuggestion && ( + )} + {primarySuggestion && !voiceFirstIdle && ( Try this @@ -2226,6 +2805,39 @@ export default function ConverseScreen() { )} + {liveVoiceReady && (liveVoiceConnected || liveVoiceConnecting || liveCaption || liveHeardLine) && ( + + + Live with Ace + + {liveVoiceConnecting + ? 'Joining' + : liveRemoteSpeaking + ? 'Speaking' + : liveVoiceMicEnabled + ? 'Listening' + : liveVoiceConnected + ? 'Standing by' + : 'Fallback'} + + + {liveHeardLine ? ( + + You: {liveHeardLine} + + ) : null} + {liveCaption ? ( + + Ace: {liveCaption} + + ) : ( + + Ace stays in the room. Planning stays open. Booking opens only when you confirm. + + )} + + )} + {recentTurns.length > 0 && ( @@ -2263,8 +2875,11 @@ export default function ConverseScreen() { const sym = pendingPlanRef.current?.fiatSymbol ?? currencySymbol; const code = pendingPlanRef.current?.fiatCode ?? currencyCode; const plan = pendingPlanRef.current?.plan ?? []; - // Family members from Phase 1 profile (non-sensitive — names + relationships only) - const tpFam = (pendingPlanRef.current?.travelProfile?.familyMembers as Array<{ name: string; relationship: string; railcard?: string }> | undefined) ?? []; + const requestedTravellers = ((pendingPlanRef.current?.travelProfile?.requestedTravellers as Array<{ + name: string; + relationship: 'adult' | 'child' | 'infant'; + }> | undefined) ?? []) + .filter((traveller) => traveller.name?.trim()); const sharedMembers = ((pendingPlanRef.current?.travelProfile?.sharedTravelUnit as { members?: Array<{ name: string; role: 'self' | 'partner' | 'adult' | 'child' | 'infant' }>; } | undefined)?.members ?? []) @@ -2273,11 +2888,11 @@ export default function ConverseScreen() { name: member.role === 'self' ? 'You' : member.name, relationship: member.role === 'child' || member.role === 'infant' ? member.role : 'adult' as const, })); - const passengers = sharedMembers.length > 0 - ? sharedMembers - : tpFam.length > 0 - ? [{ name: 'You', relationship: 'adult' as const }, ...tpFam] - : null; + const passengers = requestedTravellers.length > 0 + ? requestedTravellers + : sharedMembers.length > 0 + ? sharedMembers + : [{ name: 'You', relationship: 'adult' as const }]; const adultCount = passengers?.filter((p) => p.relationship !== 'child' && p.relationship !== 'infant').length ?? 1; const childCount = passengers?.filter((p) => p.relationship === 'child').length ?? 0; const travellerCount = passengers?.length ?? 1; @@ -2336,6 +2951,12 @@ export default function ConverseScreen() { const companionSummary = travellerCount > 1 ? `${travellerCount} travellers${childCount > 0 ? `, including ${childCount} child${childCount === 1 ? '' : 'ren'}` : ''}` : null; + const passengerDisplay = buildPassengerDisplay(passengers, travellerCount > 1 ? `${travellerCount} travellers` : 'You'); + const bookingScopeLabel = bookingMode === 'shared' + ? `Holding this for ${preferredTravelUnit?.name ?? companionSummary ?? passengerDisplay}.` + : travellerCount > 1 + ? `Holding this just on your device for ${passengerDisplay}.` + : 'Holding this just for you.'; const hotel = plan.length === 1 ? plan[0]?.hotelDetails?.bestOption : undefined; const isMultiLeg = plan.length > 1; const itinerarySummary = isMultiLeg @@ -2363,6 +2984,15 @@ export default function ConverseScreen() { {confirmMeta ? ( {confirmMeta} ) : null} + + 1 ? 'people-outline' : 'person-outline'} size={13} color="#cfe4f6" /> + + + {travellerCount > 1 ? `Travellers: ${passengerDisplay}` : 'Traveller: You'} + + {bookingScopeLabel} + + {brief && brief.cues.length > 0 && ( {brief.cues.map((cue, idx) => ( @@ -2478,14 +3108,14 @@ export default function ConverseScreen() { )} - {textFallbackVisible && ( + {(textFallbackVisible || isError) && ( Continue in text - Keep it short. Ace will still handle the rest. + Type the trip once. Ace will still handle the route, timing, and confirmation from here. member.name.trim()); + if (familyMembers.length === 0) return null; + + const selfMember = makeBlankMember('self'); + selfMember.name = firstName(profile.legalName) || 'You'; + + const members = familyMembers.map((member) => { + const role: TravelUnitMemberRole = + member.relationship === 'adult' + ? familyMembers.length === 1 ? 'partner' : 'adult' + : member.relationship; + return { + ...makeBlankMember(role), + name: member.name.trim(), + role, + state: 'guest' as const, + }; + }); + + return { + id: makeTravelUnitId(), + name: familyMembers.length === 1 ? `${selfMember.name} & ${firstName(familyMembers[0].name)}` : 'Family', + type: familyMembers.length === 1 ? 'couple' : 'family', + members: [selfMember, ...members], + primaryPayerMemberId: selfMember.id, + notes: '', + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }; +} + export default function TravelTogetherScreen() { const params = useLocalSearchParams<{ invite?: string }>(); const [units, setUnits] = useState([]); @@ -109,14 +146,16 @@ export default function TravelTogetherScreen() { const [remoteNotice, setRemoteNotice] = useState(null); const [remoteBusy, setRemoteBusy] = useState(false); const [remoteUnits, setRemoteUnits] = useState([]); + const [profile, setProfile] = useState(null); const refresh = useCallback(() => { - void Promise.all([loadTravelUnits(), loadPreferredTravelUnitId(), loadCredentials()]) - .then(async ([loadedUnits, loadedPreferredId, creds]) => { + void Promise.all([loadTravelUnits(), loadPreferredTravelUnitId(), loadCredentials(), loadProfileRaw().catch(() => null)]) + .then(async ([loadedUnits, loadedPreferredId, creds, loadedProfile]) => { const nextUnits = loadedUnits.length > 0 ? loadedUnits : [makeBlankUnit()]; setUnits(nextUnits); setPreferredId(loadedPreferredId); setAgentCreds(creds); + setProfile(loadedProfile); setSelectedId((current) => current && nextUnits.some((unit) => unit.id === current) ? current : nextUnits[0]?.id ?? null); if (creds) { @@ -164,6 +203,36 @@ export default function TravelTogetherScreen() { setSelectedId(nextUnit.id); }, []); + const savedCompanions = (profile?.familyMembers ?? []).filter((member) => member.name.trim()); + + const handleImportSavedCompanions = useCallback(async () => { + if (!profile) { + Alert.alert('Profile not ready', 'Save your family or companion details first so Ace can import them here.'); + return; + } + + const importedUnit = buildTravelUnitFromProfile(profile); + if (!importedUnit) { + Alert.alert('No saved companions yet', 'Add a partner, child, or family member to your profile first.'); + return; + } + + setSaving(true); + try { + await saveTravelUnit(importedUnit); + await savePreferredTravelUnitId(importedUnit.id); + replaceSelectedUnit(importedUnit); + setPreferredId(importedUnit.id); + setRemoteNotice( + importedUnit.type === 'couple' + ? `${importedUnit.name} is now your usual shared travel unit.` + : 'Your saved family is now ready as a shared travel unit.', + ); + } finally { + setSaving(false); + } + }, [profile, replaceSelectedUnit]); + const handleSave = useCallback(async () => { if (!selectedUnit) return; @@ -411,6 +480,22 @@ export default function TravelTogetherScreen() { + {savedCompanions.length > 0 && ( + + + + Bring in saved companions + + Ace already knows {savedCompanions.length === 1 ? `${savedCompanions[0].name} from your profile` : `${savedCompanions.length} saved companions from your profile`}. Import them once so shared booking and natural voice phrasing work cleanly. + + + { void handleImportSavedCompanions(); }} style={styles.importBtn} disabled={saving}> + {saving ? 'Working...' : 'Import'} + + + + )} + Link another Ace account @@ -741,6 +826,13 @@ const styles = StyleSheet.create({ infoCard: { backgroundColor: C.surface2, borderWidth: 1, borderColor: C.borderMd, borderRadius: 14, padding: 14, marginBottom: 18 }, infoTitle: { fontSize: 13, fontWeight: '700', color: C.textPrimary, marginBottom: 6 }, infoBody: { fontSize: 12, color: C.textSecondary, lineHeight: 18 }, + importCard: { backgroundColor: '#0a1624', borderWidth: 1, borderColor: '#17314a', borderRadius: 14, padding: 14, marginBottom: 18 }, + importHeader: { flexDirection: 'row', alignItems: 'center', gap: 12 }, + importCopy: { flex: 1 }, + importTitle: { fontSize: 13, fontWeight: '700', color: '#edf6ff', marginBottom: 6 }, + importBody: { fontSize: 12, color: '#a9bfd4', lineHeight: 18 }, + importBtn: { borderRadius: 12, backgroundColor: '#1d4ed8', paddingHorizontal: 14, paddingVertical: 10, alignItems: 'center', justifyContent: 'center' }, + importBtnText: { fontSize: 12, fontWeight: '700', color: '#eff6ff' }, linkCard: { backgroundColor: '#120914', borderWidth: 1, borderColor: '#4a1d67', borderRadius: 14, padding: 14, marginBottom: 18 }, linkTitle: { fontSize: 13, fontWeight: '700', color: '#f5d0fe', marginBottom: 6 }, linkBody: { fontSize: 12, color: '#e9d5ff', lineHeight: 18, marginBottom: 12 }, diff --git a/apps/meridian/app/(main)/wallet.tsx b/apps/meridian/app/(main)/wallet.tsx index 62930b5..515c36d 100644 --- a/apps/meridian/app/(main)/wallet.tsx +++ b/apps/meridian/app/(main)/wallet.tsx @@ -157,11 +157,12 @@ export default function WalletScreen() { ))} - {/* Add card — opens Setup Intent flow once @stripe/stripe-react-native is installed */} - Alert.alert('Add Card', 'Complete Stripe setup to enable native card entry.\nInstall @stripe/stripe-react-native and wire createSetupIntent() from lib/api.ts.')}> - - Add payment card - + + + + Saved cards are being finished for this device. For now, keep your Ace balance topped up and bookings can still move cleanly. + + {/* Top-up instructions */} @@ -248,16 +249,16 @@ const styles = StyleSheet.create({ }, cardLabel: { fontSize: 14, color: '#d1d5db', fontWeight: '500' }, defaultBadge: { fontSize: 11, color: '#6366f1', marginTop: 2 }, - addCardBtn: { + addCardNotice: { flexDirection: 'row', - alignItems: 'center', + alignItems: 'flex-start', gap: 8, paddingTop: 12, marginTop: 4, borderTopWidth: 1, borderTopColor: '#1a1a1a', }, - addCardText: { fontSize: 14, color: '#6366f1' }, + addCardNoticeText: { flex: 1, fontSize: 13, lineHeight: 18, color: '#8ea4bc' }, depositCard: { backgroundColor: '#111111', borderRadius: 14, diff --git a/apps/meridian/app/_layout.tsx b/apps/meridian/app/_layout.tsx index ffe155c..db4167f 100644 --- a/apps/meridian/app/_layout.tsx +++ b/apps/meridian/app/_layout.tsx @@ -3,6 +3,7 @@ import { Stack, useRouter } from 'expo-router'; import { GestureHandlerRootView } from 'react-native-gesture-handler'; import { StatusBar } from 'expo-status-bar'; import * as Notifications from 'expo-notifications'; +import '../lib/livekitGlobals'; import { trackClientEvent } from '../lib/telemetry'; export default function RootLayout() { diff --git a/apps/meridian/ios/Ace/Info.plist b/apps/meridian/ios/Ace/Info.plist new file mode 100644 index 0000000..8ba0f87 --- /dev/null +++ b/apps/meridian/ios/Ace/Info.plist @@ -0,0 +1,97 @@ + + + + + CADisableMinimumFrameDurationOnPhone + + CFBundleDevelopmentRegion + $(DEVELOPMENT_LANGUAGE) + CFBundleDisplayName + Ace + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + $(PRODUCT_BUNDLE_PACKAGE_TYPE) + CFBundleShortVersionString + 1.0.0 + CFBundleSignature + ???? + CFBundleURLTypes + + + CFBundleURLSchemes + + meridian + so.agentpay.meridian + + + + CFBundleVersion + 11 + ITSAppUsesNonExemptEncryption + + LSMinimumSystemVersion + 12.0 + LSRequiresIPhoneOS + + NSAppTransportSecurity + + NSAllowsArbitraryLoads + + NSAllowsLocalNetworking + + + NSFaceIDUsageDescription + Ace uses Face ID to protect your stored credentials. + NSCameraUsageDescription + Ace uses the camera only when you add travel documents or scan booking details. + NSLocationAlwaysAndWhenInUseUsageDescription + Ace uses your location to find your nearest station. + NSLocationAlwaysUsageDescription + Allow $(PRODUCT_NAME) to access your location + NSLocationWhenInUseUsageDescription + Ace uses your location to find your nearest station and suggest quick routes. + NSMicrophoneUsageDescription + Ace uses your microphone to capture voice requests. + NSMotionUsageDescription + Allow $(PRODUCT_NAME) to access your device motion + NSSpeechRecognitionUsageDescription + Ace uses speech recognition to understand your requests. + NSUserActivityTypes + + $(PRODUCT_BUNDLE_IDENTIFIER).expo.index_route + + UILaunchStoryboardName + SplashScreen + UIRequiredDeviceCapabilities + + arm64 + + UIRequiresFullScreen + + UIStatusBarStyle + UIStatusBarStyleDefault + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + UIUserInterfaceStyle + Dark + UIViewControllerBasedStatusBarAppearance + + + diff --git a/apps/meridian/lib/api.ts b/apps/meridian/lib/api.ts index 0ff3253..cc92688 100644 --- a/apps/meridian/lib/api.ts +++ b/apps/meridian/lib/api.ts @@ -11,6 +11,7 @@ const BRO_KEY = process.env.EXPO_PUBLIC_BRO_KEY ?? ''; function requiresBroKey(path: string): boolean { return ( path.startsWith('/api/concierge/') || + path.startsWith('/api/payments/') || path.startsWith('/api/shared-travel/') || path.startsWith('/api/trip-rooms/') || path.startsWith('/api/support/') @@ -92,6 +93,11 @@ export interface BroTravelProfile { currentLat?: number; currentLon?: number; familyMembers?: BroFamilyMember[]; + requestedTravellers?: Array<{ + name: string; + relationship: 'adult' | 'child' | 'infant'; + }>; + requestedTravelMode?: 'solo' | 'shared'; sharedTravelUnit?: { id: string; name: string; @@ -310,10 +316,53 @@ export interface ConciergeExecutionSnapshot { updatedAt: string | null; } +export interface VoiceSessionConfig { + mode: 'batch' | 'live'; + transport: 'http' | 'webrtc'; + provider: 'batch_proxy' | 'livekit' | 'openai_realtime'; + ready: boolean; + planningToolsAvailableDuringConversation: boolean; + bookingToolsLockedUntilConfirm: boolean; + supportsInterruptions: boolean; + supportsServerVad: boolean; + premiumVoice: 'elevenlabs' | 'system' | 'none'; + fallback: { + stt: 'whisper_proxy'; + tts: 'elevenlabs_http' | 'none'; + }; + diagnostics: string[]; +} + +export interface LiveVoiceSessionCredentials { + mode: 'live'; + provider: 'livekit'; + serverUrl: string; + roomName: string; + participantIdentity: string; + participantToken: string; + expiresAt: string; + planningToolsAvailableDuringConversation: true; + bookingToolsLockedUntilConfirm: true; +} + export async function getConciergeExecution(jobId: string): Promise { return apiFetch(`/api/concierge/executions/${jobId}`); } +export async function getVoiceSessionConfig(): Promise { + return apiFetch('/api/voice/session'); +} + +export async function createLiveVoiceSession(params: { + hirerId: string; + sessionId: string; +}): Promise { + return apiFetch('/api/voice/session/connect', { + method: 'POST', + body: JSON.stringify(params), + }); +} + /** Discover agents (text search) */ export async function discoverAgents(params: { q?: string; diff --git a/apps/meridian/lib/liveVoice.ts b/apps/meridian/lib/liveVoice.ts new file mode 100644 index 0000000..3bd6f35 --- /dev/null +++ b/apps/meridian/lib/liveVoice.ts @@ -0,0 +1,175 @@ +import { AudioSession } from '@livekit/react-native'; +import { Audio } from 'expo-av'; +import { Platform } from 'react-native'; +import { + ConnectionState, + Room, + RoomEvent, + type ChatMessage, + type Participant, + type RoomConnectOptions, + type TranscriptionSegment, +} from 'livekit-client'; +import { createLiveVoiceSession } from './api'; + +async function prepareLiveVoiceAudio(): Promise { + const { granted } = await Audio.requestPermissionsAsync(); + if (!granted) { + throw new Error('Microphone permission denied.'); + } + + if (Platform.OS === 'android') { + try { + await Audio.setAudioModeAsync({ + allowsRecordingIOS: false, + playThroughEarpieceAndroid: false, + staysActiveInBackground: false, + }); + } catch { + // Non-fatal. LiveKit will still attempt to claim the communication route. + } + + await new Promise((resolve) => setTimeout(resolve, 80)); + } +} + +async function releaseLiveVoiceAudio(): Promise { + if (Platform.OS !== 'android') return; + + try { + await Audio.setAudioModeAsync({ + allowsRecordingIOS: false, + playThroughEarpieceAndroid: false, + staysActiveInBackground: false, + }); + } catch { + // Ignore teardown errors. The next voice path will reset the route again. + } +} + +export type LiveVoiceCallbacks = { + onConnected?: () => void; + onDisconnected?: () => void; + onReconnecting?: () => void; + onReconnected?: () => void; + onConnectionStateChanged?: (state: ConnectionState) => void; + onRemoteSpeakingChanged?: (speaking: boolean) => void; + onParticipantConnected?: (participant: Participant) => void; + onParticipantDisconnected?: (participant: Participant) => void; + onChatMessage?: (message: ChatMessage, participant?: Participant) => void; + onTranscriptionReceived?: (segments: TranscriptionSegment[], participant?: Participant) => void; + onError?: (error: Error) => void; +}; + +export type LiveVoiceConnection = { + room: Room; + roomName: string; + participantIdentity: string; + disconnect: () => Promise; + setMicrophoneEnabled: (enabled: boolean) => Promise; +}; + +export async function connectLiveVoiceSession(params: { + hirerId: string; + sessionId: string; + callbacks?: LiveVoiceCallbacks; +}): Promise { + const credentials = await createLiveVoiceSession({ + hirerId: params.hirerId, + sessionId: params.sessionId, + }); + + const room = new Room({ + adaptiveStream: false, + dynacast: false, + stopLocalTrackOnUnpublish: true, + }); + + const callbacks = params.callbacks; + const emitRemoteSpeaking = () => { + const remoteSpeaking = room.activeSpeakers.some((participant) => participant.identity !== room.localParticipant.identity); + callbacks?.onRemoteSpeakingChanged?.(remoteSpeaking); + }; + + room + .on(RoomEvent.Connected, () => { + callbacks?.onConnected?.(); + emitRemoteSpeaking(); + }) + .on(RoomEvent.Disconnected, () => { + callbacks?.onDisconnected?.(); + callbacks?.onRemoteSpeakingChanged?.(false); + }) + .on(RoomEvent.Reconnecting, () => { + callbacks?.onReconnecting?.(); + }) + .on(RoomEvent.Reconnected, () => { + callbacks?.onReconnected?.(); + emitRemoteSpeaking(); + }) + .on(RoomEvent.ConnectionStateChanged, (state) => { + callbacks?.onConnectionStateChanged?.(state); + }) + .on(RoomEvent.ActiveSpeakersChanged, () => { + emitRemoteSpeaking(); + }) + .on(RoomEvent.ParticipantConnected, (participant) => { + callbacks?.onParticipantConnected?.(participant); + }) + .on(RoomEvent.ParticipantDisconnected, (participant) => { + callbacks?.onParticipantDisconnected?.(participant); + emitRemoteSpeaking(); + }) + .on(RoomEvent.ChatMessage, (message, participant) => { + callbacks?.onChatMessage?.(message, participant); + }) + .on(RoomEvent.TranscriptionReceived, (segments, participant) => { + callbacks?.onTranscriptionReceived?.(segments, participant); + }); + + try { + await prepareLiveVoiceAudio(); + await AudioSession.startAudioSession(); + room.prepareConnection(credentials.serverUrl, credentials.participantToken); + const connectOptions: RoomConnectOptions = { + autoSubscribe: true, + }; + await room.connect(credentials.serverUrl, credentials.participantToken, connectOptions); + await room.localParticipant.setMicrophoneEnabled(true, { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }); + } catch (error: any) { + callbacks?.onError?.(error instanceof Error ? error : new Error(String(error))); + try { + await room.disconnect(); + } catch {} + await AudioSession.stopAudioSession().catch(() => {}); + await releaseLiveVoiceAudio(); + throw error; + } + + return { + room, + roomName: credentials.roomName, + participantIdentity: credentials.participantIdentity, + disconnect: async () => { + callbacks?.onRemoteSpeakingChanged?.(false); + try { + await room.localParticipant.setMicrophoneEnabled(false).catch(() => {}); + await room.disconnect(); + } finally { + await AudioSession.stopAudioSession().catch(() => {}); + await releaseLiveVoiceAudio(); + } + }, + setMicrophoneEnabled: async (enabled: boolean) => { + await room.localParticipant.setMicrophoneEnabled(enabled, { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }); + }, + }; +} diff --git a/apps/meridian/lib/livekitGlobals.ts b/apps/meridian/lib/livekitGlobals.ts new file mode 100644 index 0000000..786b0b0 --- /dev/null +++ b/apps/meridian/lib/livekitGlobals.ts @@ -0,0 +1,11 @@ +import { registerGlobals } from '@livekit/react-native'; + +let registered = false; + +export function ensureLiveKitGlobals(): void { + if (registered) return; + registerGlobals(); + registered = true; +} + +ensureLiveKitGlobals(); diff --git a/apps/meridian/lib/telemetry.ts b/apps/meridian/lib/telemetry.ts index 671a684..d51fe2d 100644 --- a/apps/meridian/lib/telemetry.ts +++ b/apps/meridian/lib/telemetry.ts @@ -15,6 +15,10 @@ function enrichMetadata(metadata?: Record): Record { if (!playbackStarted && !settled) { settled = true; @@ -311,10 +313,10 @@ export async function speakBro(text: string, options?: SpeakBroOptions): Promise try { sound.setOnAudioSampleReceived(null); } catch {} options?.onMeter?.(0); void stopActivePlayback().finally(() => - reject(new Error('TTS playback watchdog: no audio start within 1200ms')), + reject(new Error(`TTS playback watchdog: no audio start within ${PLAYBACK_START_TIMEOUT_MS}ms`)), ); } - }, 1200); + }, PLAYBACK_START_TIMEOUT_MS); sound.setOnPlaybackStatusUpdate((status) => { if (status.isLoaded && status.isPlaying && !playbackStarted) { @@ -334,6 +336,18 @@ export async function speakBro(text: string, options?: SpeakBroOptions): Promise } } }); + + void sound.playAsync().catch((error) => { + clearTimeout(watchdog); + if (settled) return; + settled = true; + stopMeterPulse(); + try { sound.setOnAudioSampleReceived(null); } catch {} + options?.onMeter?.(0); + void stopActivePlayback().finally(() => + reject(error instanceof Error ? error : new Error(String(error))), + ); + }); }); void trackClientEvent({ event: 'tts_played', @@ -347,8 +361,6 @@ export async function speakBro(text: string, options?: SpeakBroOptions): Promise } catch (error: any) { options?.onMeter?.(0); await stopActivePlayback(); - // Silent fail — no robotic iOS fallback. Premium product stays silent - // rather than breaking the voice character. void trackClientEvent({ event: 'tts_failed', screen: 'voice', diff --git a/apps/meridian/lib/voiceSession.ts b/apps/meridian/lib/voiceSession.ts new file mode 100644 index 0000000..307a9da --- /dev/null +++ b/apps/meridian/lib/voiceSession.ts @@ -0,0 +1,42 @@ +import { getVoiceSessionConfig, type VoiceSessionConfig } from './api'; + +export const DEFAULT_VOICE_SESSION_CONFIG: VoiceSessionConfig = { + mode: 'batch', + transport: 'http', + provider: 'batch_proxy', + ready: true, + planningToolsAvailableDuringConversation: true, + bookingToolsLockedUntilConfirm: true, + supportsInterruptions: false, + supportsServerVad: false, + premiumVoice: 'elevenlabs', + fallback: { + stt: 'whisper_proxy', + tts: 'elevenlabs_http', + }, + diagnostics: ['session_manifest_unavailable_batch_retained'], +}; + +let cachedVoiceSessionConfig: VoiceSessionConfig | null = null; + +export async function loadVoiceSessionConfig(forceRefresh = false): Promise { + if (!forceRefresh && cachedVoiceSessionConfig) { + return cachedVoiceSessionConfig; + } + + try { + cachedVoiceSessionConfig = await getVoiceSessionConfig(); + return cachedVoiceSessionConfig; + } catch { + cachedVoiceSessionConfig = DEFAULT_VOICE_SESSION_CONFIG; + return cachedVoiceSessionConfig; + } +} + +export function getCachedVoiceSessionConfig(): VoiceSessionConfig { + return cachedVoiceSessionConfig ?? DEFAULT_VOICE_SESSION_CONFIG; +} + +export function isLiveVoiceSession(config: VoiceSessionConfig): boolean { + return config.mode === 'live' && config.ready; +} diff --git a/apps/meridian/package.json b/apps/meridian/package.json index 561c184..fe4a83c 100644 --- a/apps/meridian/package.json +++ b/apps/meridian/package.json @@ -21,7 +21,11 @@ "postinstall": "node scripts/patch-expo-modules.js" }, "dependencies": { + "@config-plugins/react-native-webrtc": "^14.0.0", "@expo/vector-icons": "~14.0.4", + "@livekit/react-native": "^2.10.0", + "@livekit/react-native-expo-plugin": "^1.0.2", + "@livekit/react-native-webrtc": "^144.0.0", "@react-native-async-storage/async-storage": "1.23.1", "@react-three/fiber": "^8.18.0", "@shopify/react-native-skia": "1.5.0", @@ -48,6 +52,7 @@ "expo-speech": "~13.0.0", "expo-status-bar": "~2.0.0", "expo-updates": "~0.27.5", + "livekit-client": "^2.18.1", "react": "18.3.1", "react-dom": "^18.3.1", "react-native": "0.76.9",