UI Automation (UIA) Development

Learn how to work with Microsoft UI Automation in NVDA

Overview

Microsoft UI Automation (UIA) is a modern accessibility API that provides programmatic access to UI elements. NVDA uses UIA to support modern Windows applications, Windows Terminal, Microsoft Office, and other UIA-aware applications.

Architecture

Core Components

The NVDA UIA implementation consists of several key modules:

• UIAHandler: Core UIA initialization and management
• UIAHandler.remote: Remote operations for better performance
• UIAHandler.browseMode: Browse mode for UIA documents
• UIAHandler.customAnnotations: Custom annotation support
• UIAHandler.customProps: Custom property definitions

UIA Element Hierarchy

from comInterfaces import UIAutomationClient as UIA

# Get root element
rootElement = UIAHandler.handler.rootElement

# Navigate the tree
focusElement = UIAHandler.handler.clientObject.GetFocusedElement()
parent = focusElement.GetParent()
firstChild = focusElement.FindFirst(UIA.TreeScope_Children, condition)

# Access properties
name = element.CurrentName
controlType = element.CurrentControlType
value = element.GetCurrentPropertyValue(UIA.UIA_ValueValuePropertyId)

Working with UIA in NVDA

Checking if UIA Should Be Used

from UIAHandler import (
    shouldUseUIAInMSWord,
    isUIAWindow,
    goodUIAWindowClassNames,
    badUIAWindowClassNames,
)

def isUIAWindow(windowHandle):
    """Check if a window should use UIA."""
    # Get window class name
    className = winUser.getClassName(windowHandle)
    
    # Check good/bad lists
    if className in goodUIAWindowClassNames:
        return True
    if className in badUIAWindowClassNames:
        return False
    
    # Check if window exposes UIA
    try:
        element = UIAHandler.handler.clientObject.ElementFromHandle(windowHandle)
        return element is not None
    except COMError:
        return False

def shouldUseUIAInMSWord(appModule):
    """Determine if UIA should be used in Microsoft Word."""
    allow = config.conf["UIA"]["allowInMSWord"]
    
    if allow == AllowUiaInMSWord.ALWAYS:
        return True
    
    # Check if old object model is available
    canUseOlderApproach = bool(appModule.helperLocalBindingHandle)
    if not canUseOlderApproach:
        return True  # Must use UIA
    
    # Check Windows version and Office version
    if winVersion.getWinVer() < winVersion.WIN11:
        return False
    
    if allow == AllowUiaInMSWord.WHERE_SUITABLE:
        officeVersion = tuple(int(x) for x in appModule.productVersion.split(".")[:3])
        return officeVersion >= (16, 0, 15000)
    
    return False

Control Type Mapping

from UIAHandler import UIAControlTypesToNVDARoles
import controlTypes

# Map UIA control types to NVDA roles
UIAControlTypesToNVDARoles = {
    UIA.UIA_ButtonControlTypeId: controlTypes.Role.BUTTON,
    UIA.UIA_CheckBoxControlTypeId: controlTypes.Role.CHECKBOX,
    UIA.UIA_ComboBoxControlTypeId: controlTypes.Role.COMBOBOX,
    UIA.UIA_EditControlTypeId: controlTypes.Role.EDITABLETEXT,
    UIA.UIA_HyperlinkControlTypeId: controlTypes.Role.LINK,
    UIA.UIA_ImageControlTypeId: controlTypes.Role.GRAPHIC,
    UIA.UIA_ListControlTypeId: controlTypes.Role.LIST,
    UIA.UIA_ListItemControlTypeId: controlTypes.Role.LISTITEM,
    UIA.UIA_MenuControlTypeId: controlTypes.Role.POPUPMENU,
    UIA.UIA_MenuItemControlTypeId: controlTypes.Role.MENUITEM,
    UIA.UIA_TreeControlTypeId: controlTypes.Role.TREEVIEW,
    UIA.UIA_TreeItemControlTypeId: controlTypes.Role.TREEVIEWITEM,
    # ... more mappings
}

def getRole(uiaElement):
    """Get NVDA role for a UIA element."""
    controlType = uiaElement.CurrentControlType
    return UIAControlTypesToNVDARoles.get(controlType, controlTypes.Role.UNKNOWN)

UIA Events

Event Registration

import UIAHandler
from comtypes import COMObject
from comInterfaces.UIAutomationClient import IUIAutomationEventHandler

class UIAEventHandler(COMObject):
    """Handler for UIA events."""
    
    _com_interfaces_ = [IUIAutomationEventHandler]
    
    def __init__(self, eventId, element, scope=UIA.TreeScope_Element):
        self.eventId = eventId
        super().__init__()
        
        # Register for event
        UIAHandler.handler.clientObject.AddAutomationEventHandler(
            eventId,
            element,
            scope,
            None,  # cache request
            self,
        )
    
    def HandleAutomationEvent(self, sender, eventId):
        """Called when the event fires."""
        try:
            # Process event
            obj = UIAHandler.getNVDAObjectFromUIAElement(sender)
            if obj:
                eventHandler.queueEvent("UIA_event", obj)
        except Exception:
            log.error("Error handling UIA event", exc_info=True)

# Common event IDs
focusChangedEventId = UIA.UIA_AutomationFocusChangedEventId
propertyChangedEventId = UIA.UIA_AutomationPropertyChangedEventId
liveRegionChangedEventId = UIA.UIA_LiveRegionChangedEventId

Event Mapping

UIAEventIdsToNVDAEventNames = {
    UIA.UIA_LiveRegionChangedEventId: "liveRegionChange",
    UIA.UIA_SelectionItem_ElementSelectedEventId: "UIA_elementSelected",
    UIA.UIA_MenuOpenedEventId: "gainFocus",
    UIA.UIA_ToolTipOpenedEventId: "UIA_toolTipOpened",
    UIA.UIA_Window_WindowOpenedEventId: "UIA_window_windowOpen",
    UIA.UIA_SystemAlertEventId: "UIA_systemAlert",
    UIA.UIA_Text_TextSelectionChangedEventId: "caret",
}

UIAPropertyIdsToNVDAEventNames = {
    UIA.UIA_NamePropertyId: "nameChange",
    UIA.UIA_ValueValuePropertyId: "valueChange",
    UIA.UIA_ExpandCollapseExpandCollapseStatePropertyId: "stateChange",
    UIA.UIA_ToggleToggleStatePropertyId: "stateChange",
    UIA.UIA_IsEnabledPropertyId: "stateChange",
}

Text Patterns

Using TextPattern

from comInterfaces.UIAutomationClient import IUIAutomationTextPattern

class UIATextInfo(textInfos.TextInfo):
    """TextInfo implementation using UIA TextPattern."""
    
    def __init__(self, obj, position, _rangeObj=None):
        super().__init__(obj, position)
        self._rangeObj = _rangeObj
        
        if not self._rangeObj:
            # Get text pattern from element
            pattern = obj.UIAElement.GetCurrentPattern(UIA.UIA_TextPatternId)
            self._pattern = pattern.QueryInterface(IUIAutomationTextPattern)
            
            # Create range based on position
            if position == textInfos.POSITION_FIRST:
                self._rangeObj = self._pattern.DocumentRange
                self._rangeObj.MoveEndpointByRange(
                    UIA.TextPatternRangeEndpoint_End,
                    self._rangeObj,
                    UIA.TextPatternRangeEndpoint_Start,
                )
            elif position == textInfos.POSITION_LAST:
                self._rangeObj = self._pattern.DocumentRange
                self._rangeObj.MoveEndpointByRange(
                    UIA.TextPatternRangeEndpoint_Start,
                    self._rangeObj,
                    UIA.TextPatternRangeEndpoint_End,
                )
            elif position == textInfos.POSITION_CARET:
                selection = self._pattern.GetSelection()
                if selection.length > 0:
                    self._rangeObj = selection.GetElement(0)
                else:
                    self._rangeObj = self._pattern.DocumentRange
    
    def _get_text(self):
        """Get text in this range."""
        return self._rangeObj.GetText(-1)  # -1 = all text
    
    def move(self, unit, direction, endPoint=None):
        """Move by the specified unit."""
        if endPoint == "start":
            endpoint = UIA.TextPatternRangeEndpoint_Start
        elif endPoint == "end":
            endpoint = UIA.TextPatternRangeEndpoint_End
        else:
            # Collapse to start, move start, then expand
            self.collapse()
            endpoint = UIA.TextPatternRangeEndpoint_Start
        
        # Map NVDA unit to UIA unit
        uiaUnit = NVDAUnitsToUIAUnits.get(unit, UIA.TextUnit_Character)
        
        # Perform the move
        moved = self._rangeObj.MoveEndpointByUnit(
            endpoint,
            uiaUnit,
            direction,
        )
        
        return moved

# Unit mapping
NVDAUnitsToUIAUnits = {
    textInfos.UNIT_CHARACTER: UIA.TextUnit_Character,
    textInfos.UNIT_WORD: UIA.TextUnit_Word,
    textInfos.UNIT_LINE: UIA.TextUnit_Line,
    textInfos.UNIT_PARAGRAPH: UIA.TextUnit_Paragraph,
    textInfos.UNIT_PAGE: UIA.TextUnit_Page,
    textInfos.UNIT_STORY: UIA.TextUnit_Document,
}

Formatting Information

def getFormatFieldAtRange(self, textRange):
    """Get formatting for a text range."""
    formatField = textInfos.FormatField()
    
    # Font information
    try:
        fontName = textRange.GetAttributeValue(UIA.UIA_FontNameAttributeId)
        if fontName != UIA.UIA_NotSupported:
            formatField["font-name"] = fontName
        
        fontSize = textRange.GetAttributeValue(UIA.UIA_FontSizeAttributeId)
        if fontSize != UIA.UIA_NotSupported:
            formatField["font-size"] = f"{fontSize}pt"
        
        bold = textRange.GetAttributeValue(UIA.UIA_IsBoldAttributeId)
        if bold:
            formatField["bold"] = True
        
        italic = textRange.GetAttributeValue(UIA.UIA_IsItalicAttributeId)
        if italic:
            formatField["italic"] = True
    
    except COMError:
        pass
    
    return formatField

Remote Operations

Remote operations allow batching multiple UIA calls for better performance:

from UIAHandler import remote

class RemoteOperation:
    """Execute multiple UIA operations in a batch."""
    
    def __init__(self):
        self.operations = []
    
    def getText(self, element):
        """Queue getting text from an element."""
        op = remote.createRemoteOperation()
        op.addCall("GetText", element)
        self.operations.append(op)
        return len(self.operations) - 1
    
    def execute(self):
        """Execute all queued operations."""
        results = []
        for op in self.operations:
            try:
                result = remote.executeRemoteOperation(op)
                results.append(result)
            except Exception:
                results.append(None)
        return results

Custom Annotations

from UIAHandler import customAnnotations

def getCustomAnnotations(element):
    """Get custom annotations from a UIA element."""
    try:
        # Get custom annotation pattern
        pattern = element.GetCurrentPattern(UIA.UIA_CustomAnnotationPatternId)
        if not pattern:
            return []
        
        annotations = []
        annotationObjects = pattern.CurrentAnnotationObjects
        
        for annotObj in annotationObjects:
            annotType = annotObj.CurrentAnnotationTypeId
            annotName = annotObj.CurrentAnnotationTypeName
            
            annotations.append({
                "type": annotType,
                "name": annotName,
            })
        
        return annotations
    
    except COMError:
        return []

Best Practices

Performance

Use remote operations to batch UIA calls and reduce COM overhead

Error Handling

Always catch COMError when accessing UIA properties - elements may become invalid

Caching

Use UIA cache requests to pre-fetch commonly accessed properties

Event Filtering

Only register for events you actually need - UIA can generate many events

UIA elements can become stale if the underlying UI changes. Always be prepared to handle exceptions.

Common Patterns

Finding Elements

def findDescendant(element, condition):
    """Find a descendant element matching a condition."""
    return element.FindFirst(UIA.TreeScope_Descendants, condition)

def createPropertyCondition(propertyId, value):
    """Create a condition for a property value."""
    return UIAHandler.handler.clientObject.CreatePropertyCondition(
        propertyId,
        value,
    )

# Example: Find a button by name
buttonCondition = UIAHandler.handler.clientObject.CreateAndCondition(
    createPropertyCondition(UIA.UIA_ControlTypePropertyId, UIA.UIA_ButtonControlTypeId),
    createPropertyCondition(UIA.UIA_NamePropertyId, "OK"),
)
button = findDescendant(rootElement, buttonCondition)

Walking the Tree

def walkTree(element, callback, maxDepth=10, currentDepth=0):
    """Walk UIA tree calling callback for each element."""
    if currentDepth >= maxDepth:
        return
    
    # Process current element
    if callback(element) is False:
        return  # Stop if callback returns False
    
    # Get children
    try:
        walker = UIAHandler.handler.clientObject.RawViewWalker
        child = walker.GetFirstChildElement(element)
        
        while child:
            walkTree(child, callback, maxDepth, currentDepth + 1)
            child = walker.GetNextSiblingElement(child)
    
    except COMError:
        pass

Reference

Key Modules

• UIAHandler: Core UIA implementation
• UIAHandler.types: Type definitions and constants
• UIAHandler.utils: Utility functions
• UIAHandler.remote: Remote operations
• comInterfaces.UIAutomationClient: COM interfaces

Important Constants

# Control types
UIA.UIA_ButtonControlTypeId
UIA.UIA_EditControlTypeId
UIA.UIA_DocumentControlTypeId

# Property IDs
UIA.UIA_NamePropertyId
UIA.UIA_ValueValuePropertyId
UIA.UIA_IsEnabledPropertyId

# Pattern IDs
UIA.UIA_TextPatternId
UIA.UIA_ValuePatternId
UIA.UIA_SelectionPatternId

# Tree scope
UIA.TreeScope_Element
UIA.TreeScope_Children
UIA.TreeScope_Descendants

Configuration

# Check UIA settings
config.conf["UIA"]["enabled"]
config.conf["UIA"]["allowInMSWord"]
config.conf["UIA"]["allowInChromium"]

Developer Guide

See the NVDA Developer Guide for complete plugin development information

Microsoft UIA Documentation

Official Microsoft documentation for UI Automation