> ## Documentation Index
> Fetch the complete documentation index at: https://mintlify.com/nvaccess/nvda/llms.txt
> Use this file to discover all available pages before exploring further.
# UI Automation (UIA) Development
> Learn how to work with Microsoft UI Automation in NVDA
## Overview
Microsoft UI Automation (UIA) is a modern accessibility API that provides programmatic access to UI elements. NVDA uses UIA to support modern Windows applications, Windows Terminal, Microsoft Office, and other UIA-aware applications.
## Architecture
### Core Components
The NVDA UIA implementation consists of several key modules:
* **UIAHandler**: Core UIA initialization and management
* **UIAHandler.remote**: Remote operations for better performance
* **UIAHandler.browseMode**: Browse mode for UIA documents
* **UIAHandler.customAnnotations**: Custom annotation support
* **UIAHandler.customProps**: Custom property definitions
### UIA Element Hierarchy
```python theme={null}
from comInterfaces import UIAutomationClient as UIA
# Get root element
rootElement = UIAHandler.handler.rootElement
# Navigate the tree
focusElement = UIAHandler.handler.clientObject.GetFocusedElement()
parent = focusElement.GetParent()
firstChild = focusElement.FindFirst(UIA.TreeScope_Children, condition)
# Access properties
name = element.CurrentName
controlType = element.CurrentControlType
value = element.GetCurrentPropertyValue(UIA.UIA_ValueValuePropertyId)
```
## Working with UIA in NVDA
### Checking if UIA Should Be Used
```python theme={null}
from UIAHandler import (
shouldUseUIAInMSWord,
isUIAWindow,
goodUIAWindowClassNames,
badUIAWindowClassNames,
)
def isUIAWindow(windowHandle):
"""Check if a window should use UIA."""
# Get window class name
className = winUser.getClassName(windowHandle)
# Check good/bad lists
if className in goodUIAWindowClassNames:
return True
if className in badUIAWindowClassNames:
return False
# Check if window exposes UIA
try:
element = UIAHandler.handler.clientObject.ElementFromHandle(windowHandle)
return element is not None
except COMError:
return False
def shouldUseUIAInMSWord(appModule):
"""Determine if UIA should be used in Microsoft Word."""
allow = config.conf["UIA"]["allowInMSWord"]
if allow == AllowUiaInMSWord.ALWAYS:
return True
# Check if old object model is available
canUseOlderApproach = bool(appModule.helperLocalBindingHandle)
if not canUseOlderApproach:
return True # Must use UIA
# Check Windows version and Office version
if winVersion.getWinVer() < winVersion.WIN11:
return False
if allow == AllowUiaInMSWord.WHERE_SUITABLE:
officeVersion = tuple(int(x) for x in appModule.productVersion.split(".")[:3])
return officeVersion >= (16, 0, 15000)
return False
```
### Control Type Mapping
```python theme={null}
from UIAHandler import UIAControlTypesToNVDARoles
import controlTypes
# Map UIA control types to NVDA roles
UIAControlTypesToNVDARoles = {
UIA.UIA_ButtonControlTypeId: controlTypes.Role.BUTTON,
UIA.UIA_CheckBoxControlTypeId: controlTypes.Role.CHECKBOX,
UIA.UIA_ComboBoxControlTypeId: controlTypes.Role.COMBOBOX,
UIA.UIA_EditControlTypeId: controlTypes.Role.EDITABLETEXT,
UIA.UIA_HyperlinkControlTypeId: controlTypes.Role.LINK,
UIA.UIA_ImageControlTypeId: controlTypes.Role.GRAPHIC,
UIA.UIA_ListControlTypeId: controlTypes.Role.LIST,
UIA.UIA_ListItemControlTypeId: controlTypes.Role.LISTITEM,
UIA.UIA_MenuControlTypeId: controlTypes.Role.POPUPMENU,
UIA.UIA_MenuItemControlTypeId: controlTypes.Role.MENUITEM,
UIA.UIA_TreeControlTypeId: controlTypes.Role.TREEVIEW,
UIA.UIA_TreeItemControlTypeId: controlTypes.Role.TREEVIEWITEM,
# ... more mappings
}
def getRole(uiaElement):
"""Get NVDA role for a UIA element."""
controlType = uiaElement.CurrentControlType
return UIAControlTypesToNVDARoles.get(controlType, controlTypes.Role.UNKNOWN)
```
## UIA Events
### Event Registration
```python theme={null}
import UIAHandler
from comtypes import COMObject
from comInterfaces.UIAutomationClient import IUIAutomationEventHandler
class UIAEventHandler(COMObject):
"""Handler for UIA events."""
_com_interfaces_ = [IUIAutomationEventHandler]
def __init__(self, eventId, element, scope=UIA.TreeScope_Element):
self.eventId = eventId
super().__init__()
# Register for event
UIAHandler.handler.clientObject.AddAutomationEventHandler(
eventId,
element,
scope,
None, # cache request
self,
)
def HandleAutomationEvent(self, sender, eventId):
"""Called when the event fires."""
try:
# Process event
obj = UIAHandler.getNVDAObjectFromUIAElement(sender)
if obj:
eventHandler.queueEvent("UIA_event", obj)
except Exception:
log.error("Error handling UIA event", exc_info=True)
# Common event IDs
focusChangedEventId = UIA.UIA_AutomationFocusChangedEventId
propertyChangedEventId = UIA.UIA_AutomationPropertyChangedEventId
liveRegionChangedEventId = UIA.UIA_LiveRegionChangedEventId
```
### Event Mapping
```python theme={null}
UIAEventIdsToNVDAEventNames = {
UIA.UIA_LiveRegionChangedEventId: "liveRegionChange",
UIA.UIA_SelectionItem_ElementSelectedEventId: "UIA_elementSelected",
UIA.UIA_MenuOpenedEventId: "gainFocus",
UIA.UIA_ToolTipOpenedEventId: "UIA_toolTipOpened",
UIA.UIA_Window_WindowOpenedEventId: "UIA_window_windowOpen",
UIA.UIA_SystemAlertEventId: "UIA_systemAlert",
UIA.UIA_Text_TextSelectionChangedEventId: "caret",
}
UIAPropertyIdsToNVDAEventNames = {
UIA.UIA_NamePropertyId: "nameChange",
UIA.UIA_ValueValuePropertyId: "valueChange",
UIA.UIA_ExpandCollapseExpandCollapseStatePropertyId: "stateChange",
UIA.UIA_ToggleToggleStatePropertyId: "stateChange",
UIA.UIA_IsEnabledPropertyId: "stateChange",
}
```
## Text Patterns
### Using TextPattern
```python theme={null}
from comInterfaces.UIAutomationClient import IUIAutomationTextPattern
class UIATextInfo(textInfos.TextInfo):
"""TextInfo implementation using UIA TextPattern."""
def __init__(self, obj, position, _rangeObj=None):
super().__init__(obj, position)
self._rangeObj = _rangeObj
if not self._rangeObj:
# Get text pattern from element
pattern = obj.UIAElement.GetCurrentPattern(UIA.UIA_TextPatternId)
self._pattern = pattern.QueryInterface(IUIAutomationTextPattern)
# Create range based on position
if position == textInfos.POSITION_FIRST:
self._rangeObj = self._pattern.DocumentRange
self._rangeObj.MoveEndpointByRange(
UIA.TextPatternRangeEndpoint_End,
self._rangeObj,
UIA.TextPatternRangeEndpoint_Start,
)
elif position == textInfos.POSITION_LAST:
self._rangeObj = self._pattern.DocumentRange
self._rangeObj.MoveEndpointByRange(
UIA.TextPatternRangeEndpoint_Start,
self._rangeObj,
UIA.TextPatternRangeEndpoint_End,
)
elif position == textInfos.POSITION_CARET:
selection = self._pattern.GetSelection()
if selection.length > 0:
self._rangeObj = selection.GetElement(0)
else:
self._rangeObj = self._pattern.DocumentRange
def _get_text(self):
"""Get text in this range."""
return self._rangeObj.GetText(-1) # -1 = all text
def move(self, unit, direction, endPoint=None):
"""Move by the specified unit."""
if endPoint == "start":
endpoint = UIA.TextPatternRangeEndpoint_Start
elif endPoint == "end":
endpoint = UIA.TextPatternRangeEndpoint_End
else:
# Collapse to start, move start, then expand
self.collapse()
endpoint = UIA.TextPatternRangeEndpoint_Start
# Map NVDA unit to UIA unit
uiaUnit = NVDAUnitsToUIAUnits.get(unit, UIA.TextUnit_Character)
# Perform the move
moved = self._rangeObj.MoveEndpointByUnit(
endpoint,
uiaUnit,
direction,
)
return moved
# Unit mapping
NVDAUnitsToUIAUnits = {
textInfos.UNIT_CHARACTER: UIA.TextUnit_Character,
textInfos.UNIT_WORD: UIA.TextUnit_Word,
textInfos.UNIT_LINE: UIA.TextUnit_Line,
textInfos.UNIT_PARAGRAPH: UIA.TextUnit_Paragraph,
textInfos.UNIT_PAGE: UIA.TextUnit_Page,
textInfos.UNIT_STORY: UIA.TextUnit_Document,
}
```
### Formatting Information
```python theme={null}
def getFormatFieldAtRange(self, textRange):
"""Get formatting for a text range."""
formatField = textInfos.FormatField()
# Font information
try:
fontName = textRange.GetAttributeValue(UIA.UIA_FontNameAttributeId)
if fontName != UIA.UIA_NotSupported:
formatField["font-name"] = fontName
fontSize = textRange.GetAttributeValue(UIA.UIA_FontSizeAttributeId)
if fontSize != UIA.UIA_NotSupported:
formatField["font-size"] = f"{fontSize}pt"
bold = textRange.GetAttributeValue(UIA.UIA_IsBoldAttributeId)
if bold:
formatField["bold"] = True
italic = textRange.GetAttributeValue(UIA.UIA_IsItalicAttributeId)
if italic:
formatField["italic"] = True
except COMError:
pass
return formatField
```
## Remote Operations
Remote operations allow batching multiple UIA calls for better performance:
```python theme={null}
from UIAHandler import remote
class RemoteOperation:
"""Execute multiple UIA operations in a batch."""
def __init__(self):
self.operations = []
def getText(self, element):
"""Queue getting text from an element."""
op = remote.createRemoteOperation()
op.addCall("GetText", element)
self.operations.append(op)
return len(self.operations) - 1
def execute(self):
"""Execute all queued operations."""
results = []
for op in self.operations:
try:
result = remote.executeRemoteOperation(op)
results.append(result)
except Exception:
results.append(None)
return results
```
## Custom Annotations
```python theme={null}
from UIAHandler import customAnnotations
def getCustomAnnotations(element):
"""Get custom annotations from a UIA element."""
try:
# Get custom annotation pattern
pattern = element.GetCurrentPattern(UIA.UIA_CustomAnnotationPatternId)
if not pattern:
return []
annotations = []
annotationObjects = pattern.CurrentAnnotationObjects
for annotObj in annotationObjects:
annotType = annotObj.CurrentAnnotationTypeId
annotName = annotObj.CurrentAnnotationTypeName
annotations.append({
"type": annotType,
"name": annotName,
})
return annotations
except COMError:
return []
```
## Best Practices
Use remote operations to batch UIA calls and reduce COM overhead
Always catch COMError when accessing UIA properties - elements may become invalid
Use UIA cache requests to pre-fetch commonly accessed properties
Only register for events you actually need - UIA can generate many events
UIA elements can become stale if the underlying UI changes. Always be prepared to handle exceptions.
## Common Patterns
### Finding Elements
```python theme={null}
def findDescendant(element, condition):
"""Find a descendant element matching a condition."""
return element.FindFirst(UIA.TreeScope_Descendants, condition)
def createPropertyCondition(propertyId, value):
"""Create a condition for a property value."""
return UIAHandler.handler.clientObject.CreatePropertyCondition(
propertyId,
value,
)
# Example: Find a button by name
buttonCondition = UIAHandler.handler.clientObject.CreateAndCondition(
createPropertyCondition(UIA.UIA_ControlTypePropertyId, UIA.UIA_ButtonControlTypeId),
createPropertyCondition(UIA.UIA_NamePropertyId, "OK"),
)
button = findDescendant(rootElement, buttonCondition)
```
### Walking the Tree
```python theme={null}
def walkTree(element, callback, maxDepth=10, currentDepth=0):
"""Walk UIA tree calling callback for each element."""
if currentDepth >= maxDepth:
return
# Process current element
if callback(element) is False:
return # Stop if callback returns False
# Get children
try:
walker = UIAHandler.handler.clientObject.RawViewWalker
child = walker.GetFirstChildElement(element)
while child:
walkTree(child, callback, maxDepth, currentDepth + 1)
child = walker.GetNextSiblingElement(child)
except COMError:
pass
```
## Reference
### Key Modules
* `UIAHandler`: Core UIA implementation
* `UIAHandler.types`: Type definitions and constants
* `UIAHandler.utils`: Utility functions
* `UIAHandler.remote`: Remote operations
* `comInterfaces.UIAutomationClient`: COM interfaces
### Important Constants
```python theme={null}
# Control types
UIA.UIA_ButtonControlTypeId
UIA.UIA_EditControlTypeId
UIA.UIA_DocumentControlTypeId
# Property IDs
UIA.UIA_NamePropertyId
UIA.UIA_ValueValuePropertyId
UIA.UIA_IsEnabledPropertyId
# Pattern IDs
UIA.UIA_TextPatternId
UIA.UIA_ValuePatternId
UIA.UIA_SelectionPatternId
# Tree scope
UIA.TreeScope_Element
UIA.TreeScope_Children
UIA.TreeScope_Descendants
```
### Configuration
```python theme={null}
# Check UIA settings
config.conf["UIA"]["enabled"]
config.conf["UIA"]["allowInMSWord"]
config.conf["UIA"]["allowInChromium"]
```
See the NVDA Developer Guide for complete plugin development information
Official Microsoft documentation for UI Automation