From e74cd7e94db6cce187a535271768fa260dff8d83 Mon Sep 17 00:00:00 2001 From: nab5 Date: Sat, 31 Oct 2026 19:54:40 +0530 Subject: [PATCH 1/2] Add Do Not Disturb, volume, status and find voice commands Say "turn on Do Not Disturb" or "turn off dnd", "volume up", "volume down", "mute" or "unmute", "battery" or "status", "open settings" or "open Backtalk settings". "Find [control]" moves focus to a matching control on screen. Do Not Disturb uses the notification policy API, and asks for access the first time. Wi-Fi, hotspot and mobile data aren't included, as Android has no supported way for an app to switch them. --- differences.md | 13 + lint.xml | 15 + talkback/src/main/AndroidManifest.xml | 2 + .../talkback/TalkBackService.java | 2 + .../voicecommands/SystemVoiceCommands.kt | 455 ++++++++++++++++++ .../voicecommands/VoiceCommandProcessor.java | 25 +- .../main/res/values/strings_voicecommands.xml | 30 ++ .../training/VoiceCommandAndHelpConfigs.java | 2 + .../actor/voicecommands/DndDirectionTest.kt | 55 +++ 9 files changed, 598 insertions(+), 1 deletion(-) create mode 100644 talkback/src/main/java/com/google/android/accessibility/talkback/actor/voicecommands/SystemVoiceCommands.kt create mode 100644 talkback/src/test/java/com/google/android/accessibility/talkback/actor/voicecommands/DndDirectionTest.kt diff --git a/differences.md b/differences.md index 2565d0a87..b86bc44d7 100644 --- a/differences.md +++ b/differences.md @@ -432,6 +432,19 @@ To turn the circle menu on, go to **Backtalk menu** and turn on **Circle menu**. Say "call Mom", or "call" followed by a phone number, to place a call. Backtalk reads back who it will call and asks you to say yes, then calls at once. Add "on speaker" to start the call on the speaker. If two contacts match, Backtalk reads their names and you say a number. Say "answer the call" or "pick up the call" to answer, "decline" to turn a call away, and "hang up" to end one. The first time, Backtalk asks for the phone, contacts and answer-calls permissions. Ending a call needs Android 9 or later. Turning the speaker on or off during a call isn't included, because Android doesn't let apps like Backtalk do it. +### System controls + +Backtalk adds voice commands for settings that Android lets apps change: +* **Do Not Disturb**: Say "turn on Do Not Disturb" or "turn on DND" to silence notifications, and "turn off DND" to restore them. If notification policy access has not been granted yet, Backtalk asks for permission and opens the setting. +* **System settings**: Say "open settings" or "settings" to open the Android system Settings app. +* **Backtalk settings**: Say "open Backtalk settings" or "Backtalk settings" to open screen reader preferences directly. +* **Device status and battery**: Say "battery" to hear the battery level and charging state, or "status" to hear the time, network, and device status. +* **Volume**: Say "volume up", "volume down", "mute", or "unmute". + +### Finding controls on screen + +Say "find [name]", "focus [name]", "search for [name]", or "locate [name]" (for example, "find play button", "focus search", "locate cancel") to search the screen and move accessibility focus directly to the matching control. + ## Screen and brightness ### Brightness reading control diff --git a/lint.xml b/lint.xml index 6efa1f0c4..a5a347e70 100644 --- a/lint.xml +++ b/lint.xml @@ -6,6 +6,21 @@ Keep MissingTranslation enabled globally and suppress it only for these exact resources. --> + + + + + + + + + + + + + + + diff --git a/talkback/src/main/AndroidManifest.xml b/talkback/src/main/AndroidManifest.xml index 51f794b08..ca8502588 100644 --- a/talkback/src/main/AndroidManifest.xml +++ b/talkback/src/main/AndroidManifest.xml @@ -16,6 +16,8 @@ + + diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java b/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java index 286780a28..9e3e7e1f7 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/TalkBackService.java @@ -2344,6 +2344,8 @@ public long getSpeechCacheWindowCounts() { voiceCommandProcessor = new VoiceCommandProcessor(this, accessibilityFocusMonitor, selectorController, analytics); + voiceCommandProcessor.setBatteryMonitor(batteryMonitor); + voiceCommandProcessor.setStatusReader(statusReader); voiceCommandActor = new VoiceCommandActor( this, diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/actor/voicecommands/SystemVoiceCommands.kt b/talkback/src/main/java/com/google/android/accessibility/talkback/actor/voicecommands/SystemVoiceCommands.kt new file mode 100644 index 000000000..5de93677e --- /dev/null +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/actor/voicecommands/SystemVoiceCommands.kt @@ -0,0 +1,455 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.android.accessibility.talkback.actor.voicecommands + +import android.accessibilityservice.AccessibilityService +import android.app.NotificationManager +import android.content.Context +import android.content.Intent +import android.media.AudioManager +import android.provider.Settings +import android.view.accessibility.AccessibilityNodeInfo +import com.google.android.accessibility.talkback.Feedback +import com.google.android.accessibility.talkback.Pipeline +import com.google.android.accessibility.talkback.R +import com.google.android.accessibility.talkback.actor.voicecommands.VoiceCommandActor.RECOGNITION_SPEECH_DELAY_MS +import com.google.android.accessibility.talkback.monitor.BatteryMonitor +import com.google.android.accessibility.talkback.preference.TalkBackPreferencesActivity +import com.google.android.accessibility.talkback.status.StatusReader +import com.google.android.accessibility.utils.Performance.EVENT_ID_UNTRACKED +import com.google.android.accessibility.utils.SettingsUtils +import com.google.android.accessibility.utils.output.FeedbackItem +import com.google.android.accessibility.utils.output.SpeechController.SpeakOptions + +/** + * Handles system-level voice commands such as: + * - Directly opening System Settings ("open settings") + * - Opening Backtalk settings ("open backtalk settings") + * - Toggling Do Not Disturb ("turn on/off dnd") + * - Device status and battery inquiries ("battery", "status", "time") + * - Volume adjustments ("volume up", "volume down", "mute", "unmute") + * - Finding a control on screen ("find play button") + */ +class SystemVoiceCommands( + private val context: Context, + private var pipeline: Pipeline.FeedbackReturner? = null, + var batteryMonitor: BatteryMonitor? = null, + var statusReader: StatusReader? = null +) { + + fun setPipeline(pipeline: Pipeline.FeedbackReturner?) { + this.pipeline = pipeline + } + + /** + * Attempts to handle the spoken command as a system or app command. + * Returns true if the command was recognized and handled. + */ + fun handleCommand(command: String, isPartialResult: Boolean): Boolean { + val norm = VoiceCommandFuzzyMatcher.normalize(command) + if (norm.isBlank()) return false + + // 1. Backtalk Settings (requires explicit mention of Backtalk or TalkBack) + if (handleBacktalkSettings(norm)) { + return true + } + + // 2. System Settings ("open settings", "settings") + if (handleSystemSettings(norm)) { + return true + } + + // 3. Do Not Disturb + if (handleDoNotDisturb(norm)) { + return true + } + + // 4. Battery & Device Status + if (handleStatusAndBattery(norm)) { + return true + } + + // 5. Volume Controls + if (handleVolume(norm)) { + return true + } + + // 6. Find / Focus on screen element ("find play button", "focus search", "locate cancel") + if (handleFindOnScreen(norm, command)) { + return true + } + + return false + } + + // --- Sub-handlers --- + + /** + * Handles opening Backtalk settings. Strictly requires that the user said + * "backtalk" or "talkback" so generic "settings" or "open settings" does not trigger this. + */ + private fun handleBacktalkSettings(norm: String): Boolean { + if (!norm.contains("backtalk") && !norm.contains("talkback")) { + return false + } + + val targets = listOf( + "open backtalk settings", + "backtalk settings", + "backtalk setting", + "open talkback settings", + "talkback settings", + "talkback setting" + ) + if (VoiceCommandFuzzyMatcher.matchesAny(norm, targets, threshold = 0.85)) { + if (SettingsUtils.allowLinksOutOfSettings(context)) { + speakDelayed(context.getString(R.string.voice_commands_opening_backtalk_settings)) + val intent = Intent(context, TalkBackPreferencesActivity::class.java).apply { + addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + } + context.startActivity(intent) + } + return true + } + return false + } + + /** + * Handles opening the device's System Settings app when the user says "open settings", + * "settings", "system settings", or "device settings". + */ + private fun handleSystemSettings(norm: String): Boolean { + val settingsTargets = listOf( + "open settings", + "settings", + "launch settings", + "start settings", + "go to settings", + "system settings", + "device settings" + ) + if (VoiceCommandFuzzyMatcher.matchesAny(norm, settingsTargets, threshold = 0.88)) { + speakDelayed(context.getString(R.string.voice_commands_opening_settings)) + val intent = Intent(Settings.ACTION_SETTINGS).apply { + addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + } + context.startActivity(intent) + return true + } + return false + } + + private fun handleDoNotDisturb(norm: String): Boolean { + val isOn = dndDirection(norm) ?: return false + + val notificationManager = context.getSystemService(Context.NOTIFICATION_SERVICE) as? NotificationManager + if (notificationManager != null) { + if (notificationManager.isNotificationPolicyAccessGranted) { + if (isOn) { + notificationManager.setInterruptionFilter(NotificationManager.INTERRUPTION_FILTER_PRIORITY) + speakDelayed(context.getString(R.string.voice_commands_dnd_on)) + } else { + notificationManager.setInterruptionFilter(NotificationManager.INTERRUPTION_FILTER_ALL) + speakDelayed(context.getString(R.string.voice_commands_dnd_off)) + } + } else { + speakDelayed(context.getString(R.string.voice_commands_dnd_permission_required)) + val intent = Intent(Settings.ACTION_NOTIFICATION_POLICY_ACCESS_SETTINGS).apply { + addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + } + context.startActivity(intent) + } + return true + } + return false + } + + private fun handleStatusAndBattery(norm: String): Boolean { + val batteryTargets = listOf("battery", "battery level", "how much battery", "check battery", "battery status") + if (VoiceCommandFuzzyMatcher.matchesAny(norm, batteryTargets, threshold = 0.85)) { + val desc = batteryMonitor?.batteryStateDescription + if (!desc.isNullOrBlank()) { + speakDelayed(desc) + return true + } + } + + val statusTargets = listOf("status", "device status", "what time is it", "time", "check status") + if (VoiceCommandFuzzyMatcher.matchesAny(norm, statusTargets, threshold = 0.85)) { + val desc = statusReader?.describe() + if (!desc.isNullOrBlank()) { + speakDelayed(desc) + return true + } + } + + return false + } + + private fun handleVolume(norm: String): Boolean { + val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager ?: return false + + val isVolUp = VoiceCommandFuzzyMatcher.matchesAny(norm, listOf("volume up", "increase volume", "louder", "turn volume up"), threshold = 0.85) + val isVolDown = VoiceCommandFuzzyMatcher.matchesAny(norm, listOf("volume down", "decrease volume", "quieter", "turn volume down", "lower volume"), threshold = 0.85) + val isMute = VoiceCommandFuzzyMatcher.matchesAny(norm, listOf("mute", "mute volume", "mute audio"), threshold = 0.85) + val isUnmute = VoiceCommandFuzzyMatcher.matchesAny(norm, listOf("unmute", "unmute volume", "unmute audio"), threshold = 0.85) + + if (isVolUp) { + audioManager.adjustStreamVolume(AudioManager.STREAM_MUSIC, AudioManager.ADJUST_RAISE, AudioManager.FLAG_SHOW_UI) + announceCurrentVolume(audioManager) + return true + } + if (isVolDown) { + audioManager.adjustStreamVolume(AudioManager.STREAM_MUSIC, AudioManager.ADJUST_LOWER, AudioManager.FLAG_SHOW_UI) + announceCurrentVolume(audioManager) + return true + } + if (isMute) { + audioManager.adjustStreamVolume(AudioManager.STREAM_MUSIC, AudioManager.ADJUST_MUTE, AudioManager.FLAG_SHOW_UI) + speakDelayed(context.getString(R.string.voice_commands_muted)) + return true + } + if (isUnmute) { + audioManager.adjustStreamVolume(AudioManager.STREAM_MUSIC, AudioManager.ADJUST_UNMUTE, AudioManager.FLAG_SHOW_UI) + speakDelayed(context.getString(R.string.voice_commands_unmuted)) + return true + } + + return false + } + + private fun announceCurrentVolume(audioManager: AudioManager) { + val current = audioManager.getStreamVolume(AudioManager.STREAM_MUSIC) + val max = audioManager.getStreamMaxVolume(AudioManager.STREAM_MUSIC) + val pct = if (max > 0) (current * 100) / max else 0 + speakDelayed(context.getString(R.string.voice_commands_volume_set, pct)) + } + + private fun speakDelayed(text: CharSequence) { + val returner = pipeline ?: return + val speakOptions = SpeakOptions.create().apply { + setFlags( + FeedbackItem.FLAG_NO_HISTORY + or FeedbackItem.FLAG_FORCE_FEEDBACK_ALL + ) + } + val feedback = Feedback.speech(text, speakOptions).setDelayMs(RECOGNITION_SPEECH_DELAY_MS) + returner.returnFeedback(EVENT_ID_UNTRACKED, feedback) + } + + // --- Screen Actions: Click and Find --- + + private val FIND_PREFIX_REGEX = Regex( + "^(?:find|search\\s+for|search|locate|focus)(?:\\s+on)?(?:\\s+the)?\\s+(.+)$", + RegexOption.IGNORE_CASE + ) + + private val ROLE_SUFFIXES = listOf( + "button", "btn", "tab", "switch", "checkbox", "check box", "toggle", + "icon", "image", "link", "menu", "option", "item", "header", "heading", + "slider", "field", "text field", "input", "card" + ) + + private data class TargetSpec( + val rawTarget: String, + val baseTarget: String, + val expectedRole: String? + ) + + private fun parseTarget(rawRemainder: String): TargetSpec { + var clean = rawRemainder.trim() + if (clean.startsWith("the ", ignoreCase = true)) { + clean = clean.substring(4).trim() + } + for (role in ROLE_SUFFIXES) { + if (clean.endsWith(" $role", ignoreCase = true)) { + val base = clean.substring(0, clean.length - role.length - 1).trim() + if (base.isNotBlank()) { + return TargetSpec(clean, base, role.lowercase()) + } + } + } + return TargetSpec(clean, clean, null) + } + + private fun handleFindOnScreen(normInput: String, rawInput: String): Boolean { + val match = FIND_PREFIX_REGEX.matchEntire(normInput) ?: return false + val rawTarget = match.groupValues[1].trim() + if (rawTarget.length < 2) return false + + val spec = parseTarget(rawTarget) + val candidate = findBestNodeOnScreen(spec) + if (candidate == null) { + speakDelayed(context.getString(R.string.voice_commands_element_not_found, spec.rawTarget)) + return true + } + + candidate.node.performAction(AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS) + speakDelayed(context.getString(R.string.voice_commands_found, candidate.label)) + return true + } + + private data class NodeCandidate( + val node: AccessibilityNodeInfo, + val score: Double, + val label: String + ) + + private fun findBestNodeOnScreen(spec: TargetSpec): NodeCandidate? { + val accessService = context as? AccessibilityService ?: return null + val root = accessService.rootInActiveWindow + ?: accessService.windows.firstOrNull { + it.isFocused || it.type == android.view.accessibility.AccessibilityWindowInfo.TYPE_APPLICATION + }?.root + ?: return null + + val queue = ArrayDeque() + queue.add(root) + var inspected = 0 + var bestCandidate: NodeCandidate? = null + + val targetFull = spec.rawTarget.lowercase() + val targetBase = spec.baseTarget.lowercase() + val expectedRole = spec.expectedRole + + while (queue.isNotEmpty() && inspected < 1000) { + val node = queue.removeFirst() + inspected++ + + val isVisible = node.isVisibleToUser + if (isVisible) { + val text = node.text?.toString()?.trim() + val desc = node.contentDescription?.toString()?.trim() + val viewId = node.viewIdResourceName?.substringAfterLast('/')?.trim() + val className = node.className?.toString() ?: "" + val isClickable = node.isClickable || node.parent?.isClickable == true + val isCheckable = node.isCheckable + + val label = when { + !desc.isNullOrBlank() -> desc + !text.isNullOrBlank() -> text + !viewId.isNullOrBlank() -> viewId + else -> null + } + + if (!label.isNullOrBlank()) { + val normLabel = label.lowercase() + var score = 0.0 + + // Exact matches + if (normLabel == targetFull) { + score = 1.0 + } else if (normLabel == targetBase) { + score = 0.95 + } else if (normLabel.startsWith(targetFull) || normLabel.endsWith(targetFull)) { + score = 0.85 + } else if (normLabel.startsWith(targetBase) || normLabel.endsWith(targetBase)) { + score = 0.80 + } else if (normLabel.contains(targetFull)) { + score = 0.75 + } else if (normLabel.contains(targetBase)) { + score = 0.70 + } else { + val sim = maxOf( + VoiceCommandFuzzyMatcher.similarity(targetFull, normLabel), + VoiceCommandFuzzyMatcher.similarity(targetBase, normLabel) + ) + if (sim >= 0.72) { + score = sim * 0.85 + } + } + + if (score > 0.0) { + // Role bonus + if (expectedRole != null) { + val matchesRole = when (expectedRole) { + "button", "btn" -> className.contains("Button", ignoreCase = true) || isClickable + "switch", "toggle", "checkbox", "check box" -> isCheckable || className.contains("Switch", ignoreCase = true) + "tab" -> className.contains("Tab", ignoreCase = true) + "image", "icon" -> className.contains("Image", ignoreCase = true) + "field", "input", "text field" -> node.isEditable || className.contains("Edit", ignoreCase = true) + else -> false + } + if (matchesRole) { + score += 0.15 + } + } + + if (isClickable) score += 0.10 + if (isVisible) score += 0.05 + + if (bestCandidate == null || score > bestCandidate.score) { + bestCandidate = NodeCandidate(node, score, label) + } + } + } + } + + for (i in 0 until node.childCount) { + val child = node.getChild(i) + if (child != null) { + queue.add(child) + } + } + } + + return bestCandidate?.takeIf { it.score >= 0.65 } + } +} + +private val DND_ON_TARGETS = listOf( + "turn on dnd", + "turn on do not disturb", + "enable dnd", + "enable do not disturb", + "dnd on", + "do not disturb on" +) +private val DND_OFF_TARGETS = listOf( + "turn off dnd", + "turn off do not disturb", + "disable dnd", + "disable do not disturb", + "dnd off", + "do not disturb off" +) + +private val DND_OFF_WORDS = setOf("off", "disable") +private val DND_ON_WORDS = setOf("on", "enable") + +/** + * Tells whether [norm] asks to turn Do Not Disturb on (true) or off (false), or null when it is + * not about Do Not Disturb. The on and off phrases differ by a word or two, so a fuzzy match + * alone finds both; the word itself decides. + */ +internal fun dndDirection(norm: String): Boolean? { + val fuzzyOn = VoiceCommandFuzzyMatcher.matchesAny(norm, DND_ON_TARGETS, threshold = 0.85) + val fuzzyOff = VoiceCommandFuzzyMatcher.matchesAny(norm, DND_OFF_TARGETS, threshold = 0.85) + if (!fuzzyOn && !fuzzyOff) return null + val words = norm.split(" ") + val saysOff = words.any { it in DND_OFF_WORDS } + val saysOn = words.any { it in DND_ON_WORDS } + return when { + // Both words, such as "turn on dnd and switch off the ringer": too unclear to act on. + saysOff && saysOn -> null + saysOff -> false + saysOn -> true + // No clear word: only act when exactly one list matches. + fuzzyOn != fuzzyOff -> fuzzyOn + else -> null + } +} diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/actor/voicecommands/VoiceCommandProcessor.java b/talkback/src/main/java/com/google/android/accessibility/talkback/actor/voicecommands/VoiceCommandProcessor.java index 0508b6ee5..8e36cd237 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/actor/voicecommands/VoiceCommandProcessor.java +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/actor/voicecommands/VoiceCommandProcessor.java @@ -92,6 +92,8 @@ import android.text.TextUtils; import androidx.core.view.accessibility.AccessibilityNodeInfoCompat; import com.google.android.accessibility.talkback.preference.TalkBackPreferencesActivity; +import com.google.android.accessibility.talkback.monitor.BatteryMonitor; +import com.google.android.accessibility.talkback.status.StatusReader; import com.google.android.accessibility.talkback.ActorState; import com.google.android.accessibility.talkback.Feedback; import com.google.android.accessibility.talkback.Interpretation; @@ -141,6 +143,7 @@ public class VoiceCommandProcessor { private ActorState actorState; private final TalkBackAnalytics analytics; private final CallVoiceCommands callVoiceCommands; + private final SystemVoiceCommands systemVoiceCommands; private boolean echoNotRecognizedTextEnabled; @@ -157,7 +160,11 @@ public class VoiceCommandProcessor { }; private static final int[] talkbackSettingCommandResArray = { - R.string.voice_commands_talkback_settings, R.string.voice_commands_talkback_setting + R.string.voice_commands_backtalk_settings, + R.string.voice_commands_backtalk_setting, + R.string.voice_commands_open_backtalk_settings, + R.string.voice_commands_talkback_settings, + R.string.voice_commands_talkback_setting }; private static final int[] finishSelectCommandResArray = { @@ -260,11 +267,21 @@ public VoiceCommandProcessor( this.selectorController = selectorController; this.analytics = analytics; this.callVoiceCommands = new CallVoiceCommands(service); + this.systemVoiceCommands = new SystemVoiceCommands(service, null, null, null); } public void setPipeline(Pipeline.FeedbackReturner pipeline) { this.pipeline = pipeline; this.callVoiceCommands.setPipeline(pipeline); + this.systemVoiceCommands.setPipeline(pipeline); + } + + public void setBatteryMonitor(BatteryMonitor batteryMonitor) { + this.systemVoiceCommands.setBatteryMonitor(batteryMonitor); + } + + public void setStatusReader(StatusReader statusReader) { + this.systemVoiceCommands.setStatusReader(statusReader); } public void setActorState(ActorState actorState) { @@ -322,6 +339,12 @@ public boolean handleSpeechCommand(String command, boolean isPartialResult) { return true; } + // Check system voice commands (dnd, status, volume, settings, find on screen) + if (systemVoiceCommands.handleCommand(command, isPartialResult)) { + analytics.onVoiceCommandEvent(VOICE_COMMAND_RECOGNIZED); + return true; + } + // Gemini voice command // command format: Gemini // TODO: When the code structure allows to test taking screenshot, add unit tests diff --git a/talkback/src/main/res/values/strings_voicecommands.xml b/talkback/src/main/res/values/strings_voicecommands.xml index 6c7e84e86..b6b907c9a 100644 --- a/talkback/src/main/res/values/strings_voicecommands.xml +++ b/talkback/src/main/res/values/strings_voicecommands.xml @@ -336,6 +336,10 @@ Type, select, and copy text Type [OK] + + Turn on Do Not Disturb + + Turn Do Not Disturb on or off. Backtalk asks for notification policy access the first time Call [contact] or Call [phone number] @@ -407,4 +411,30 @@ Ending calls by voice needs Android 9 or later Could not complete the call + + Backtalk settings + + Backtalk setting + + Open Backtalk settings + + Do Not Disturb turned on + + Do Not Disturb turned off + + Please grant Do Not Disturb access to use this voice command + + Opening Settings + + Opening Backtalk settings + + Volume %1$d percent + + Muted + + Unmuted + + Found %1$s + + Could not find %1$s on screen diff --git a/talkback/src/phone/java/com/google/android/accessibility/talkback/training/VoiceCommandAndHelpConfigs.java b/talkback/src/phone/java/com/google/android/accessibility/talkback/training/VoiceCommandAndHelpConfigs.java index fc1564430..8e4a4bf08 100644 --- a/talkback/src/phone/java/com/google/android/accessibility/talkback/training/VoiceCommandAndHelpConfigs.java +++ b/talkback/src/phone/java/com/google/android/accessibility/talkback/training/VoiceCommandAndHelpConfigs.java @@ -124,6 +124,8 @@ private VoiceCommandAndHelpConfigs() {} .addTextWithBullet(R.string.voice_commands_quick_settings) .addTextWithBullet(R.string.voice_commands_overview) .addSubText(R.string.voice_commands_help_overview_description) + .addTextWithBullet(R.string.voice_commands_help_dnd) + .addSubText(R.string.voice_commands_help_dnd_description) .addTextWithBullet(R.string.voice_commands_help_call) .addSubText(R.string.voice_commands_help_call_description) .addTextWithBullet(R.string.voice_commands_help_call_control) diff --git a/talkback/src/test/java/com/google/android/accessibility/talkback/actor/voicecommands/DndDirectionTest.kt b/talkback/src/test/java/com/google/android/accessibility/talkback/actor/voicecommands/DndDirectionTest.kt new file mode 100644 index 000000000..5d3841714 --- /dev/null +++ b/talkback/src/test/java/com/google/android/accessibility/talkback/actor/voicecommands/DndDirectionTest.kt @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + + +package com.google.android.accessibility.talkback.actor.voicecommands + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class DndDirectionTest { + + private fun direction(spoken: String) = dndDirection(VoiceCommandFuzzyMatcher.normalize(spoken)) + + @Test + fun turnsOn() { + for (phrase in listOf("turn on dnd", "turn on do not disturb", "enable do not disturb", "dnd on", "do not disturb on", "can you turn on d and d")) { + assertEquals(phrase, true, direction(phrase)) + } + } + + @Test + fun turnsOff() { + for (phrase in listOf("turn off dnd", "turn off do not disturb", "disable do not disturb", "dnd off", "do not disturb off", "turn of dnd", "please turn off do not disturb")) { + assertEquals(phrase, false, direction(phrase)) + } + } + + @Test + fun ignoresOtherCommands() { + assertNull(direction("battery")) + assertNull(direction("volume up")) + assertNull(direction("call mom")) + } + + @Test + fun doesNotGuessWhenUnclear() { + assertNull(direction("turn on dnd and turn off the ringer")) + assertNull(direction("dnd")) + assertNull(direction("do not disturb")) + } +} From ac084ca1e3eeb46af59e1c1007ba362f4f2d4a13 Mon Sep 17 00:00:00 2001 From: nab5 Date: Sun, 8 Nov 2026 18:06:21 +0530 Subject: [PATCH 2/2] Fix unmute, status and "open Backtalk settings" voice commands Phone testing found three mistakes in the system voice commands. "Unmute" muted, because the fuzzy match found "mute" inside it and the mute check came first. "Status" read the battery, because "battery status" contains "status". "Open Backtalk settings" opened Android Settings, because the speech recognizer writes "Backtalk" as "back talk" and Settings matched any sentence that contained the word "settings". These commands now match whole phrases, not words found inside a sentence. The unmute check uses the whole word, and "back talk" is read as "Backtalk". --- .../voicecommands/SystemVoiceCommands.kt | 116 ++++++++++++------ .../voicecommands/VoiceCommandFuzzyMatcher.kt | 2 + .../voicecommands/SystemCommandPhrasesTest.kt | 91 ++++++++++++++ 3 files changed, 173 insertions(+), 36 deletions(-) create mode 100644 talkback/src/test/java/com/google/android/accessibility/talkback/actor/voicecommands/SystemCommandPhrasesTest.kt diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/actor/voicecommands/SystemVoiceCommands.kt b/talkback/src/main/java/com/google/android/accessibility/talkback/actor/voicecommands/SystemVoiceCommands.kt index 5de93677e..023f8f222 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/actor/voicecommands/SystemVoiceCommands.kt +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/actor/voicecommands/SystemVoiceCommands.kt @@ -133,16 +133,7 @@ class SystemVoiceCommands( * "settings", "system settings", or "device settings". */ private fun handleSystemSettings(norm: String): Boolean { - val settingsTargets = listOf( - "open settings", - "settings", - "launch settings", - "start settings", - "go to settings", - "system settings", - "device settings" - ) - if (VoiceCommandFuzzyMatcher.matchesAny(norm, settingsTargets, threshold = 0.88)) { + if (wantsSystemSettings(norm)) { speakDelayed(context.getString(R.string.voice_commands_opening_settings)) val intent = Intent(Settings.ACTION_SETTINGS).apply { addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) @@ -179,51 +170,37 @@ class SystemVoiceCommands( } private fun handleStatusAndBattery(norm: String): Boolean { - val batteryTargets = listOf("battery", "battery level", "how much battery", "check battery", "battery status") - if (VoiceCommandFuzzyMatcher.matchesAny(norm, batteryTargets, threshold = 0.85)) { - val desc = batteryMonitor?.batteryStateDescription - if (!desc.isNullOrBlank()) { - speakDelayed(desc) - return true - } - } - - val statusTargets = listOf("status", "device status", "what time is it", "time", "check status") - if (VoiceCommandFuzzyMatcher.matchesAny(norm, statusTargets, threshold = 0.85)) { - val desc = statusReader?.describe() - if (!desc.isNullOrBlank()) { - speakDelayed(desc) - return true + val description = + when (deviceInfo(norm) ?: return false) { + DeviceInfo.BATTERY -> batteryMonitor?.batteryStateDescription + DeviceInfo.STATUS -> statusReader?.describe() } - } - - return false + if (description.isNullOrBlank()) return false + speakDelayed(description) + return true } private fun handleVolume(norm: String): Boolean { val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as? AudioManager ?: return false - val isVolUp = VoiceCommandFuzzyMatcher.matchesAny(norm, listOf("volume up", "increase volume", "louder", "turn volume up"), threshold = 0.85) - val isVolDown = VoiceCommandFuzzyMatcher.matchesAny(norm, listOf("volume down", "decrease volume", "quieter", "turn volume down", "lower volume"), threshold = 0.85) - val isMute = VoiceCommandFuzzyMatcher.matchesAny(norm, listOf("mute", "mute volume", "mute audio"), threshold = 0.85) - val isUnmute = VoiceCommandFuzzyMatcher.matchesAny(norm, listOf("unmute", "unmute volume", "unmute audio"), threshold = 0.85) + val action = volumeAction(norm) ?: return false - if (isVolUp) { + if (action == VolumeAction.UP) { audioManager.adjustStreamVolume(AudioManager.STREAM_MUSIC, AudioManager.ADJUST_RAISE, AudioManager.FLAG_SHOW_UI) announceCurrentVolume(audioManager) return true } - if (isVolDown) { + if (action == VolumeAction.DOWN) { audioManager.adjustStreamVolume(AudioManager.STREAM_MUSIC, AudioManager.ADJUST_LOWER, AudioManager.FLAG_SHOW_UI) announceCurrentVolume(audioManager) return true } - if (isMute) { + if (action == VolumeAction.MUTE) { audioManager.adjustStreamVolume(AudioManager.STREAM_MUSIC, AudioManager.ADJUST_MUTE, AudioManager.FLAG_SHOW_UI) speakDelayed(context.getString(R.string.voice_commands_muted)) return true } - if (isUnmute) { + if (action == VolumeAction.UNMUTE) { audioManager.adjustStreamVolume(AudioManager.STREAM_MUSIC, AudioManager.ADJUST_UNMUTE, AudioManager.FLAG_SHOW_UI) speakDelayed(context.getString(R.string.voice_commands_unmuted)) return true @@ -453,3 +430,70 @@ internal fun dndDirection(norm: String): Boolean? { else -> null } } + +private val SETTINGS_PHRASES = + listOf( + "open settings", + "open the settings", + "settings", + "launch settings", + "start settings", + "go to settings", + "system settings", + "device settings") + +private val BATTERY_PHRASES = + listOf("battery", "battery level", "how much battery", "check battery", "battery status") + +private val STATUS_PHRASES = + listOf("status", "device status", "check status", "what time is it", "time") + +private val VOLUME_UP_PHRASES = listOf("volume up", "increase volume", "louder", "turn volume up") + +private val VOLUME_DOWN_PHRASES = + listOf("volume down", "decrease volume", "quieter", "turn volume down", "lower volume") + +/** + * Whether [norm] is one of [phrases]. A long phrase may have one wrong letter. A sentence that only + * contains a phrase is not a match: "unmute" is not "mute", and "battery status" is not "status". + */ +private fun saysPhrase(norm: String, phrases: List): Boolean = + phrases.any { + norm == it || (it.length >= 8 && VoiceCommandFuzzyMatcher.similarity(norm, it) >= 0.85) + } + +/** Whether [norm] asks to open Android Settings. */ +internal fun wantsSystemSettings(norm: String): Boolean = saysPhrase(norm, SETTINGS_PHRASES) + +internal enum class DeviceInfo { + BATTERY, + STATUS +} + +/** Which device information [norm] asks for, or null when it asks for none. */ +internal fun deviceInfo(norm: String): DeviceInfo? = + when { + saysPhrase(norm, BATTERY_PHRASES) -> DeviceInfo.BATTERY + saysPhrase(norm, STATUS_PHRASES) -> DeviceInfo.STATUS + else -> null + } + +internal enum class VolumeAction { + UP, + DOWN, + MUTE, + UNMUTE +} + +/** Which volume change [norm] asks for, or null when it asks for none. */ +internal fun volumeAction(norm: String): VolumeAction? { + val words = norm.split(" ") + return when { + // "unmute" contains "mute", so it is told apart by the whole word. + "unmute" in words || norm.startsWith("un mute") -> VolumeAction.UNMUTE + "mute" in words && words.size <= 3 -> VolumeAction.MUTE + saysPhrase(norm, VOLUME_UP_PHRASES) -> VolumeAction.UP + saysPhrase(norm, VOLUME_DOWN_PHRASES) -> VolumeAction.DOWN + else -> null + } +} diff --git a/talkback/src/main/java/com/google/android/accessibility/talkback/actor/voicecommands/VoiceCommandFuzzyMatcher.kt b/talkback/src/main/java/com/google/android/accessibility/talkback/actor/voicecommands/VoiceCommandFuzzyMatcher.kt index 6060a8c85..13b678324 100644 --- a/talkback/src/main/java/com/google/android/accessibility/talkback/actor/voicecommands/VoiceCommandFuzzyMatcher.kt +++ b/talkback/src/main/java/com/google/android/accessibility/talkback/actor/voicecommands/VoiceCommandFuzzyMatcher.kt @@ -55,6 +55,8 @@ object VoiceCommandFuzzyMatcher { text = text.replace(Regex("\\bd and d\\b"), "dnd") text = text.replace(Regex("\\bd\\.n\\.d\\.\\b"), "dnd") + text = text.replace(Regex("\\bback talk\\b"), "backtalk") + // Strip punctuation text = PUNCTUATION_REGEX.replace(text, " ") text = MULTI_SPACE_REGEX.replace(text, " ").trim() diff --git a/talkback/src/test/java/com/google/android/accessibility/talkback/actor/voicecommands/SystemCommandPhrasesTest.kt b/talkback/src/test/java/com/google/android/accessibility/talkback/actor/voicecommands/SystemCommandPhrasesTest.kt new file mode 100644 index 000000000..6ed61b32a --- /dev/null +++ b/talkback/src/test/java/com/google/android/accessibility/talkback/actor/voicecommands/SystemCommandPhrasesTest.kt @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.android.accessibility.talkback.actor.voicecommands + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class SystemCommandPhrasesTest { + + private fun norm(spoken: String) = VoiceCommandFuzzyMatcher.normalize(spoken) + + @Test + fun unmuteIsNotMute() { + assertEquals(VolumeAction.UNMUTE, volumeAction(norm("unmute"))) + assertEquals(VolumeAction.UNMUTE, volumeAction(norm("unmute volume"))) + assertEquals(VolumeAction.UNMUTE, volumeAction(norm("please unmute"))) + assertEquals(VolumeAction.UNMUTE, volumeAction(norm("un mute"))) + assertEquals(VolumeAction.MUTE, volumeAction(norm("mute"))) + assertEquals(VolumeAction.MUTE, volumeAction(norm("mute audio"))) + } + + @Test + fun volumeUpAndDown() { + assertEquals(VolumeAction.UP, volumeAction(norm("volume up"))) + assertEquals(VolumeAction.UP, volumeAction(norm("louder"))) + assertEquals(VolumeAction.DOWN, volumeAction(norm("volume down"))) + assertEquals(VolumeAction.DOWN, volumeAction(norm("lower volume"))) + } + + @Test + fun otherSentencesAreNotVolumeCommands() { + assertNull(volumeAction(norm("open settings"))) + assertNull(volumeAction(norm("tell me about the volume of a sphere"))) + assertNull(volumeAction(norm(""))) + } + + @Test + fun statusIsNotBattery() { + assertEquals(DeviceInfo.STATUS, deviceInfo(norm("status"))) + assertEquals(DeviceInfo.STATUS, deviceInfo(norm("device status"))) + assertEquals(DeviceInfo.STATUS, deviceInfo(norm("what time is it"))) + assertEquals(DeviceInfo.BATTERY, deviceInfo(norm("battery"))) + assertEquals(DeviceInfo.BATTERY, deviceInfo(norm("battery status"))) + assertEquals(DeviceInfo.BATTERY, deviceInfo(norm("how much battery"))) + } + + @Test + fun otherSentencesAreNotDeviceInfo() { + assertNull(deviceInfo(norm("open settings"))) + assertNull(deviceInfo(norm("what is the status of my order"))) + } + + @Test + fun openSettingsMeansAndroidSettings() { + assertTrue(wantsSystemSettings(norm("open settings"))) + assertTrue(wantsSystemSettings(norm("settings"))) + assertTrue(wantsSystemSettings(norm("please open settings"))) + assertTrue(wantsSystemSettings(norm("go to settings"))) + } + + @Test + fun backtalkSettingsIsNotAndroidSettings() { + assertFalse(wantsSystemSettings(norm("open backtalk settings"))) + assertFalse(wantsSystemSettings(norm("open back talk settings"))) + assertFalse(wantsSystemSettings(norm("open talkback settings"))) + assertFalse(wantsSystemSettings(norm("open the wifi settings"))) + } + + @Test + fun backTalkInTwoWordsIsBacktalk() { + assertEquals("open backtalk settings", norm("open back talk settings")) + assertEquals("backtalk settings", norm("back talk settings")) + } +}