
    .cjt                      d Z ddlmZ ddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlZddlZddlZddlmZmZmZmZmZ ddlmZmZmZmZ  e	j        e          ZdddkdZdZ dZ!dgZ"h dZ#dZ$dZ%dZ&dldZ'dmdZ(dmdZ)dndZ*dodpd#Z+dqd&Z,drd(Z-dnd)Z.dsd+Z/d,d-dtd0Z0d1d2dud7Z1e!fdvd:Z2 ej3        d;<          dwd=            Z4 ej5        d>ej6                  Z7dmd?Z8dxdAZ9dodydCZ:dodzdDZ;dmdEZ<dFd2d{dHZ=d|dIZ>d,a?d}dKZ@d~dLZAddOZBddQZCddUZDddXZEdd[ZF G d\ d]          ZG G d^ d_          ZHddbZIddeZJddfZKddhZL G di dje          ZMdS )uv  Cua-driver backend (macOS, Windows, Linux).

Speaks MCP over stdio to `cua-driver`. The Python `mcp` SDK is async, so we
run a dedicated asyncio event loop on a background thread and marshal sync
calls through it.

The same `cua-driver call <tool>` surface (click, type_text, hotkey, drag,
scroll, screenshot, launch_app, list_apps, list_windows, get_window_state,
move_cursor, wait) works identically across macOS, Windows, and Linux —
cua-driver's PARITY matrix marks the action tools VERIFIED on macOS and
Windows in the cross-platform Rust port (`cua-driver-rs`).

Linux is the most recent runtime (X11 today, Wayland via XWayland; pure-
Wayland progress tracked upstream). It is enabled in
`check_computer_use_requirements` alongside macOS and Windows. The plumbing
in this file is OS-agnostic; per-host gaps (no DISPLAY, missing AT-SPI,
etc.) surface as specific blocked checks via `hermes computer-use doctor`
rather than failing silently.

Install:
  - **macOS**:
      /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver/scripts/install.sh)"
  - **Windows** (PowerShell):
      irm https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver/scripts/install.ps1 | iex

After install, `cua-driver` is on $PATH and supports `cua-driver mcp` (stdio
transport) which is what we invoke.

The macOS path uses private SkyLight SPIs (SLEventPostToPid,
SLPSPostEventRecordTo, _AXObserverAddNotificationAndCheckRemote) that aren't
Apple-public and can break on OS updates. The Windows path in cua-driver-rs
uses stable Win32 APIs (SendInput + UI Automation) — not subject to the
same SPI breakage class.
    )annotationsN)AnyDictListOptionalTuple)ActionResultCaptureResultComputerUseBackend	UIElementrequested_deliverynamestrokboolmessagemetaDict[str, Any]
structuredr   Optional[str]returnr	   c               Z   t          |t                    r|ni dfd} |d          }t          |t                    sd} |d          }t          |t                    sd} |d	          }	t          |	t                    sd}	 |d
          }
t          |
t                    sd}
 |d          }t          |t                    sd} |d          p
 |d          }t          |t                    sd}t          |t                    r|nd}t	          || ||||	|
|||          S )a  Build an ActionResult, lifting cua-driver's structured verdict.

    All structured fields are additive: a driver that omits
    ``structuredContent`` (or any individual field) leaves the corresponding
    ActionResult attribute ``None``, so callers and tests see unchanged
    behavior on old drivers. See the action response shape in
    cua-driver's mcp-tool-notes and NousResearch/hermes-agent#67052.
    keyr   r   r   c                `    | v r                     |           S                      |           S Nget)r   r   scs    @/home/ice/.hermes/hermes-agent/tools/computer_use/cua_backend.py_pickz"_action_result_from.<locals>._pickR   s,    "9966#;;xx}}    verifiedNeffect
escalationpathdegradedcodereason_code)r   actionr   r   r#   r$   r%   r&   r'   delivery_moder(   )r   r   r   r   )
isinstancedictr   r   r	   )r   r   r   r   r   r   r!   r#   r$   r%   r&   r'   r(   r+   r   s      `          @r    _action_result_fromr.   ?   s   " "*d33	;B       uZ  Hh%% U8__Ffc"" |$$Jj$'' 
5==DdC   uZ  Hh%% 5==0EE-00DdC    +55G*M*MW&&SWM#   r"   HERMES_CUA_DRIVER_CMD
cua-drivermcp>   full screenallscreendesktop
fullscreen)progmanworkerwprogram managershell_traywndtaskbarfinderr5   dock)z@!Desktopzgnome-shellzGNOME ShellCUA_DRIVER_RS_TELEMETRY_ENABLEDc                 v    	 ddl m}   |             pi                     d          pi S # t          $ r i cY S w xY w)zGThe ``computer_use`` config block, or ``{}`` when config is unreadable.r   load_configcomputer_use)hermes_cli.configrB   r   	ExceptionrA   s    r    _computer_use_cfgrF      sa    111111#((88>B>   			s   &) 88c                    t                                          d          } | t          |           S t          j        dk    rdS t          j        dk    rdS t
          j                            d          sdS 	 t          dd	
          5 }d|                                	                                v r	 ddd           dS 	 ddd           n# 1 swxY w Y   n# t          $ r Y nw xY wdS )u  True when Hermes should pass ``--no-overlay`` to cua-driver.

    Reads ``computer_use.no_overlay``. Default ``None`` (auto-detect):
    disable the overlay where idle CPU burn is a known failure mode —
    macOS (cursor-overlay vImage redraw loop, #28152/#47032), headless
    Linux / WSL2 / containers — and keep it on Windows / desktop Linux
    with a display. Explicit ``True`` / ``False`` overrides auto-detection.
    
no_overlayNdarwinTlinuxFDISPLAYz/proc/versionzutf-8)encoding	microsoft)rF   r   r   sysplatformosenvironopenreadlowerrE   )valfs     r    _cua_no_overlayrW      sL    


!
!,
/
/C
Cyy |xt
|wu:>>)$$ t/G444 	affhhnn....	 	 	 	 	 	 	 	.	 	 	 	 	 	 	 	 	 	 	 	 	 	 	    5s<   9C 
*C4C C CC CC 
C'&C'c                 b    t          t                                          dd                     S )u   True when Hermes should disable cua-driver telemetry for this user.

    Reads ``computer_use.cua_telemetry`` (default False → telemetry off).
    Unreadable config falls SAFE toward disabling telemetry.
    cua_telemetryF)r   rF   r    r"   r    _cua_telemetry_disabledr[      s,     %''++OUCCDDDDr"   Optional[int]c                     	 t          t                                          dd                    } n# t          t          f$ r Y dS w xY w| dk    r| ndS )u   Longest-edge cap for cua-driver screenshots, or None to leave unset.

    Reads ``computer_use.max_image_dimension`` (default 1456, matching the
    aux-vision downscale). ``0`` / negative / non-numeric → None.
    max_image_dimensioni  r   N)intrF   r   	TypeError
ValueError)dims    r    !_computer_use_max_image_dimensionrc      sg    #%%))*?FFGGz"   tt''33t#s   /2 AAbase_envOptional[Dict[str, str]]Dict[str, str]c                p    t          | | nt          j                  }t                      r
d|t          <   |S )a  Return the environment dict for spawning cua-driver.

    Starts from ``base_env`` (defaults to ``os.environ``) and, when telemetry
    is disabled (the default), injects ``CUA_DRIVER_RS_TELEMETRY_ENABLED=0``.
    When the user has opted in, the var is left untouched so cua-driver uses
    its own default. Used by every cua-driver spawn site (MCP backend, status,
    doctor, install) so the policy is applied consistently.
    N0)r-   rP   rQ   r[   _CUA_TELEMETRY_ENV_VAR)rd   envs     r    cua_driver_child_envrk      s:     8/xxRZ
@
@C   *&)"#Jr"   windowsList[Dict[str, Any]]c                D    | sdS t          d | D                       dk    S )zETrue when every window shares the same z_index (common on Linux/X11).Tc                :    h | ]}|                     d d          S )z_indexr   r   .0ws     r    	<setcomp>z)_z_index_uninformative.<locals>.<setcomp>  s&    555i##555r"      )len)rl   s    r    _z_index_uninformativerw   	  s1     t55W55566!;;r"   stdoutc                    | pd}t          j        d|          }|st          j        d|          }|sdS 	 t          |                    d          d          S # t          $ r Y dS w xY w)zParse ``xprop -root _NET_ACTIVE_WINDOW`` stdout into a window id.

    Accepts the common ``window id # 0x...`` form and falls back to the first
    hex token. Returns None for empty/unparseable output.
     zwindow id # (0x[0-9a-fA-F]+)z(0x[0-9a-fA-F]+)Nru      )researchr_   groupra   )rx   textmatchs      r    _parse_xprop_net_active_windowr     s     <RDI5t<<E 5	-t44 t5;;q>>2&&&   tts   "A 
A'&A'c                    t           j        dk    st          j                            d          sdS 	 t          j        g ddddd          } n# t          $ r Y dS w xY w| j        d	k    rdS t          | j
        pd
          S )zCBest-effort read of ``_NET_ACTIVE_WINDOW`` via xprop. Never raises.rJ   rK   N)xpropz-root_NET_ACTIVE_WINDOWT   F)capture_outputr   timeoutcheckr   rz   )rN   rO   rP   rQ   r   
subprocessrunrE   
returncoder   rx   )procs    r    _linux_x11_active_window_idr   "  s    
|wbjnnY&?&?t	~444
 
 
    tt!t)$+*;<<<s   A 
AArs   c                r    |                      dd          t          fdt          D                        S )zDReturn False for desktop/shell helper windows that capture as empty.titlerz   c              3     K   | ]R}                     |          p8                                                     |                                          V  Sd S r   )
startswithrT   )rr   pr   s     r    	<genexpr>z&_is_real_app_window.<locals>.<genexpr>8  sh         	Bu{{}}77		BB     r"   )r   any_NON_APP_WINDOW_TITLE_PREFIXES)rs   r   s    @r    _is_real_app_windowr   5  sS    EE'2E    /      r"   F)exact_targetapp_requestedr   c                  d | D             }|}|se|sct           j        dk    rSd |D             }|r|}|rAt          |          r2t                      }|"|D ]}|                    d          |k    r|c S  |r|d         S | d         S )u  Select the best window for capture from normalized list_windows output.

    Callers pass windows already sorted by ``z_index`` descending (higher =
    frontmost). When ordering is informative, keep that frontmost contract.
    For unqualified default captures (no app filter and no exact
    pid/window_id) on Linux, desktop/shell helper windows (GNOME ``ding``
    "Desktop Icons", ``@!x,y;BDHF`` backdrop helpers) are skipped first —
    they are targetable X11 windows but capture as empty. Then, when every
    remaining candidate shares the same ``z_index`` (tied or unknown, the
    common Linux/X11 case), prefer ``_NET_ACTIVE_WINDOW`` over list order
    (#58026). Exact-target captures must not pay for an ``xprop`` probe.
    c                "    g | ]}|d          
|S )
off_screenrZ   rq   s     r    
<listcomp>z*_select_capture_target.<locals>.<listcomp>P  s!    <<<AlO<!<<<r"   rJ   c                0    g | ]}t          |          |S rZ   )r   rq   s     r    r   z*_select_capture_target.<locals>.<listcomp>S  s&    EEE1.A!.D.DEQEEEr"   N	window_idr   )rN   rO   rw   r   r   )rl   r   r   
candidatespool	real_apps	active_idrs   s           r    _select_capture_targetr   >  s    $ =<W<<<JD 	! 	!#,'2I2IEE
EEE	 	D 	!*400 	!355I$ ! !Auu[))Y66  7 Aw1:r"   g      @r   
driver_cmdr   floatTuple[str, List[str]]c                  	 ddl m} t          j        | dgdd|t          j         |t                                          }n5# t          $ r( | t          t          t                    |           fcY S w xY w|j
        pd                                }|j        dk    s|s%| t          t          t                    |           fS 	 t          j        |          }n<# t          t           f$ r( | t          t          t                    |           fcY S w xY wt#          |t$                    s%| t          t          t                    |           fS |                    d          }t#          |t$                    s%| t          t          t                    |           fS |                    d	          }|                    d
          }t#          |t                    rt)          d |D                       s%| t          t          t                    |           fS t#          |t*                    r|s| t          ||           fS t-          |          s| t          ||           fS |t          ||          fS )u  Return ``(command, args)`` that spawn cua-driver's stdio MCP server.

    Surface 8 of NousResearch/hermes-agent#47072: instead of hardcoding
    ``["mcp"]`` we ask the driver itself via ``cua-driver manifest``
    (trycua/cua#1961). The manifest carries a stable ``mcp_invocation``
    pointer with both ``command`` and ``args``, so a future cua-driver
    that renames or relocates the subcommand keeps working without a
    Hermes patch.

    Falls back to ``(driver_cmd, ["mcp"])`` for older drivers that don't
    expose ``manifest``, or any indeterminate failure — the wrapper must
    not refuse to start just because the discovery hop failed.

    When ``computer_use.no_overlay`` is enabled (or auto-detected on
    Linux), ``--no-overlay`` is appended to suppress the cursor overlay
    rendering loop that can consume CPU indefinitely when idle
    (#28152, #47032).  Older drivers that don't recognise the flag will
    reject it; callers should fall back to the no-overlay invocation on
    spawn failure.
    r   _sanitize_subprocess_envmanifestTr   r   r   stdinrj   )r   rz   mcp_invocationargscommandc              3  @   K   | ]}t          |t                    V  d S r   r,   r   )rr   as     r    r   z*_resolve_mcp_invocation.<locals>.<genexpr>  s,      ,N,NAZ3-?-?,N,N,N,N,N,Nr"   )tools.environments.localr   r   r   DEVNULLrk   rE   _mcp_args_with_overlay_flaglist_CUA_DRIVER_ARGSrx   stripr   jsonloadsra   r`   r,   r-   r   r3   r   _has_path_separator)	r   r   r   r   outr   
invocationr   r   s	            r    _resolve_mcp_invocationr   a  s   2fEEEEEE~$dG$ )()=)?)?@@
 
 
  f f f6t<L7M7MZdeeeeeeef;"
#
#
%
%C!36t<L7M7MZdeeeeef:c??	" f f f6t<L7M7MZdeeeeeeefh%% f6t<L7M7MZdeeeee.//Jj$'' f6t<L7M7MZdeeeee>>&!!DnnY''GdD!! f,N,N,N,N,N)N)N f6t<L7M7MZdeeeeegs## T7 T 6t
SSSSSw'' T 6t
SSSSS
 /IIIIIs$   AA /A65A6C 6DDr   	List[str]c                f    t                      rt          |          rg | dS t          |           S )zKReturn *args* with ``--no-overlay`` appended when configured and supported.--no-overlay)rW   _cua_driver_supports_no_overlayr   )r   r   s     r    r   r     s>    
  '<ZHH '&&~&&::r"   ru   )maxsizec                    	 ddl m} t          j        | dgdddt          j         |t                                          }|j        pd|j        pdz   }d|v S # t          $ r Y d	S w xY w)
zTrue if the installed cua-driver recognises ``--no-overlay``.

    Probes ``<driver> --help`` once and caches the result.  Older
    drivers (< 0.6.x) reject unknown flags, so passing ``--no-overlay``
    would crash the MCP spawn.
    r   r   z--helpTg      @r   rz   r   F)	r   r   r   r   r   rk   rx   stderrrE   )r   r   r   	help_texts       r    r   r     s     	FEEEEE~"dC$(()=)?)?@@	
 
 
 [&B4;+<"=	**   uus   AA 
A('A(z~^\s*(?:-\s+)?\[(\d+)\]\s+(\w+)(?:\s*=\s*"([^"]*)"|\s+"([^"]*)"|\s+\((?!\d+\))([^)]*)\))?(?:\s+(?:\(\d+\)\s+)?id=([^\s\[\]]+))?c                 "    t           j        dk    S )NrI   rN   rO   rZ   r"   r    	_is_macosr     s    <8##r"   valuec                V    t           j        | v pt           j        d uot           j        | v S r   )rP   sepaltsep)r   s    r    r   r     s&    6U?Lry4Ke9KLr"   overridec           	     V   | | n$t           j                            t          d                                          }|r|gS t
          g}t           j                            d          }t          j	        dk    rX|
                    t           j                            |ddd          t           j                            |ddd          g           nY|
                    t           j                            |ddd          t           j                            |d	dd          d
dg           |S )a  Return candidate cua-driver commands in resolution order.

    ``override`` is authoritative when supplied. Otherwise a non-empty
    ``HERMES_CUA_DRIVER_CMD`` is authoritative; only when neither is set do we
    use PATH and canonical install locations.

    Desktop apps launched from Finder/Dock often inherit a narrow PATH that
    omits user-local install directories. The upstream cua-driver installer
    commonly places the binary under ``~/.local/bin`` on POSIX systems, so a
    Hermes Desktop/TUI session can otherwise filter out the `computer_use`
    tool even though `hermes computer-use doctor` succeeds from a login shell.
    Nrz   ~win32z.localbinzcua-driver.exer0   z.cargoz/opt/homebrew/bin/cua-driverz/usr/local/bin/cua-driver)rP   rQ   r   _CUA_DRIVER_CMD_ENVr   _CUA_DRIVER_DEFAULT_CMDr&   
expanduserrN   rO   extendjoin)r   
configuredr   homes       r    _candidate_cua_driver_commandsr     s    '2((
GZ\^8_8_ffhhJ  |)*J7c""D
|wGLLx0@AAGLLx==
 	 	 	 	
 	GLLx==GLLx==*'	
 	 	 	 r"   c                    t          |           D ]c}t          j                            |          }t	          |          rt          j        |          r|c S It          j        |          }|r|c S ddS )a%  Resolve the cua-driver executable for every runtime/status surface.

    A supplied override (or ``HERMES_CUA_DRIVER_CMD``) is never silently
    replaced by another binary. Otherwise resolve PATH first, then canonical
    user-local installation locations used by the official installer.
    N)r   rP   r&   r   r   shutilwhich)r   	candidateexpandedresolveds       r    resolve_cua_driver_cmdr     s     4H==    	7%%i00x(( 	 |H%%    |H--H   4r"   c                 "    t                      duS )zDTrue if `cua-driver` resolves via env, PATH, or known install paths.N)r   rZ   r"   r    cua_driver_binary_availabler   +  s    !##4//r"   g       @Optional[Dict[str, Any]]c                   t                      }|sdS 	 ddlm} t          j        |ddgdd| t          j         |t                                          }n# t          $ r Y dS w xY w|j        pd	                                }|sdS 	 t          j        |          }n# t          t          f$ r Y dS w xY wt          |t                    r|                    d	          rdS |S )
a  Run ``cua-driver check-update --json`` and return its parsed state.

    The payload mirrors the ``check_for_update`` MCP tool:
    ``{current_version, latest_version, update_available, ...}``.

    Returns ``None`` (callers should stay quiet) when the result is
    indeterminate: the binary is missing, the driver is too old to support
    the verb (it predates trycua/cua#1734), the GitHub check failed (an
    ``error`` field is set), or the output didn't parse. Best-effort; never
    raises.
    Nr   r   zcheck-updatez--jsonTr   rz   error)r   r   r   r   r   r   rk   rE   rx   r   r   r   ra   r`   r,   r-   r   )r   r   r   r   r   datas         r    cua_driver_update_checkr   0  s.    ())J tEEEEEE~2dG $ )()=)?)?@@

 

 

    tt;"
#
#
%
%C tz#	"   ttdD!! TXXg%6%6 tKs$   AA 
A%$A%B B21B2c                     t                      } | r|                     d          sdS |                     d          pd}|                     d          pd}d| d| dS )	z~One-line "an update is available" message, or ``None`` when up to date,
    indeterminate, or the driver is too old to report.update_availableNlatest_version?current_versioncua-driver z is available (you have z7); update with `hermes computer-use install --upgrade`.)r   r   )statelatestcurrents      r    cua_driver_update_nudger   \  s     $%%E 		"455 tYY'((/CFii)**1cG	@f 	@ 	@g 	@ 	@ 	@r"   Nonec                 v    t           rdS da dd} t          j        | dd                                           dS )	zEmit an update nudge at most once per process, off-thread so the
    (cached, ~20h) GitHub poll never blocks the first computer_use action.NTr   r   c                     	 t                      } n# t          $ r Y d S w xY w| rt                              d|            d S d S )Nzcomputer_use: %s)r   rE   loggerinfo)msgs    r    _runz!_maybe_nudge_update.<locals>._runu  sc    	)++CC 	 	 	FF	 	1KK*C00000	1 	1s    
zcua-driver-update-check)targetr   daemonr   r   )_update_checked	threadingThreadstart)r   s    r    _maybe_nudge_updater  m  sX      O1 1 1 1 3D  egggggr"   c                 8    t           j        dk    rd} nd} d|  dS )Nr   za  irm https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver/scripts/install.ps1 | iexzs  /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver/scripts/install.sh)"zxcua-driver is not installed. Install with one of:
  hermes computer-use install
Or run the upstream installer directly:
zW
Or run `hermes tools` and enable the Computer Use toolset to install it automatically.r   )	installers    r    cua_driver_install_hintr    sG    
|w8 		3 		a 	a 	a 	ar"   markdownList[UIElement]c           
        g }t                               |           D ]}|                    d          p@|                    d          p+|                    d          p|                    d          pd}|                    t	          t          |                    d                    |                    d          |d	                     |S )
u  Parse UIElement list from get_window_state AX tree markdown.

    Last-resort fallback for cua-driver builds that don't carry the
    canonical ``structuredContent.elements`` array (see
    ``_parse_elements_from_structured`` — Surface 2 of #47072 prefers
    that path).

    Captures the label whichever form cua-driver used: ``= "value"``,
    ``"quoted"``, ``(parenthesised)``, or ``id=Label``. Bounds always
    come back ``(0, 0, 0, 0)`` because the markdown surface doesn't
    carry them — yet another reason to prefer the structured path;
    element-index clicks don't need them (the driver resolves the index
    to a frame internally).
                rz   ru   r   r   r   r   r   )indexrolelabelbounds)_ELEMENT_LINE_REfinditerr~   appendr   r_   )r  elementsmr  s       r    _parse_elements_from_treer    s     H&&x00  

JaggajjJAGGAJJJ!''!**J	aggajj//	
 
 
 	 	 	 	 Or"   raw_elementsc           
        g }| D ]}t          |t                    s|                    d          }t          |t                    sDt          |                    d          t                    r|                    d          nd}t          |                    d          t                    r|                    d          nd}t          |                    d          t                    r|                    d          nd}d}|r	 t          |                    dd	                    t          |                    d
d	                    t          |                    dd	                    t          |                    dd	                    f}n# t
          t          f$ r d}Y nw xY w|                    d          }t          |t                    r|r|nd}	|                    t          |||||	                     |S )uG  Surface 2 of NousResearch/hermes-agent#47072: read the canonical
    ``structuredContent.elements`` array cua-driver-rs emits on every
    ``get_window_state`` response (trycua/cua#1961).

    Each entry has at minimum ``element_index``, ``role``, ``label``;
    ``frame`` (``{x, y, w, h}``) is included whenever the AT-SPI /
    AXFrame call returned usable bounds. Older code parsed the same
    information out of the markdown tree via a regex (lossy: bounds
    were always ``(0, 0, 0, 0)``) — this path preserves the real
    frame so downstream consumers (e.g. ``UIElement.center()``) work
    against pixel coordinates instead of just the index lookup.

    Unknown / malformed entries are skipped rather than failing the
    whole walk — the wrapper degrades to "fewer elements" rather than
    "no elements" on a bad row.
    element_indexr  rz   r  frameNr  xr   yrs   helement_token)r  r  r  r  r  )	r,   r-   r   r_   r   r`   ra   r  r   )
r  r  rawidxr  r  r  r  	raw_tokentokens
             r    _parse_elements_from_structuredr$    s   " !#H  #t$$ 	ggo&&#s## 	",SWWV__c"B"BJswwv$.swww/?/?$E$EM   2$.swww/?/?$F$FP   D,8 		&&		#q))**		#q))**		#q))**		#q))**	 z* & & &%&
 GGO,,	'	377OIO		4	
 
 
 	 	 	 	 Os   B
FF,+F,r   bytesTuple[int, int]c                F   |                      d          rit          |           dk    rVt                              | dd         d          }t                              | dd         d          }|dk    r
|dk    r||fS |                      d          rd}t          |           }|d	z   |k     r| |         d
k    r|dz  }| |dz            }|dz  }|dv sd|cxk    rdk    rn n@|dz   |k    rnt                              | ||dz            d          }|dk     s	||z   |k    rn{|dv ri|dk    rbt                              | |dz   |dz            d          }t                              | |dz   |dz            d          }|dk    r
|dk    r||fS n||z  }|d	z   |k     dS )zCBest-effort PNG/JPEG dimension sniffing without extra dependencies.s   PNG

   r{      bigr   s   r   	      ru   >               >                                             r	  r  )r   r   )r   rv   r_   
from_bytes)r   widthheightinmarkersegment_lens          r    _image_dimensions_from_bytesrF    s   
~~*++ !CBs2b5z511BrE
E22199!&= 
~~k"" HH!eaii1v~~QQZFFA%%)?)?)?)?4)?)?)?)?)?1uqyy..Qq1uWu==KQ!k/A"5"5    !## ^^CAa!e,<eDDFNN3q1uQU{+;UCCEqyyVaZZ$f},A/ !eaii2 4r"   	full_textTuple[str, str]c                    |                      dd          }|d         }t          |          dk    r|d         nd}||fS )z?Split get_window_state text into (summary_line, tree_markdown).
ru   r   rz   )splitrv   )rG  linessummarytrees       r    _split_tree_textrO    sC    OOD!$$EAhG5zzA~~5882DD=r"   keysTuple[Optional[str], List[str]]c                    h d}dddd}d t          j        d|           D             }g }d}|D ]4}|                    ||          }||v r|                    |           2|}5||fS )	zParse a key string like 'cmd+s' into (key, modifiers).

    Returns (key, modifiers) where key is the non-modifier key and modifiers
    is a list of modifier names (cmd, shift, option, ctrl).
    >   fnaltcmdctrlshiftoptionr   controlrU  rX  rV  )r   rT  rY  c                    g | ]<}|                                 |                                                                 =S rZ   )r   rT   )rr   r   s     r    r   z$_parse_key_combo.<locals>.<listcomp>   s7    NNN1AGGIINQWWYY__NNNr"   z[+\-]N)r|   rK  r   r  )rP  MODIFIER_NAMESKEY_ALIASESparts	modifiersr   part
normalizeds           r    _parse_key_combora    s     [ZZN#HHHKNN4(@(@NNNEI
C   __T400
''Z((((CC	>r"   c                  4    e Zd ZdZddZddZddd
ZddZdS )_AsyncBridgezNRuns one asyncio loop on a daemon thread; marshals coroutines from the caller.r   r   c                R    d | _         d | _        t          j                    | _        d S r   )_loop_threadr   Event_readyselfs    r    __init__z_AsyncBridge.__init__3  s#    :>
37o''r"   c                H     j         r j                                         rd S  j                                         d
 fd}t	          j        |dd           _          j                                           j                            d          st          d	          d S )Nr   r   c                    t          j                     _        t          j         j                    j                                         	  j                                         	  j                                         d S # t          $ r Y d S w xY w# 	  j                                         w # t          $ r Y w w xY wxY wr   )	asyncionew_event_loopre  set_event_looprh  setrun_forevercloserE   ri  s   r    r   z _AsyncBridge.start.<locals>._run=  s     /11DJ"4:...KOO
&&(((J$$&&&&&    DDJ$$&&&&    DsB   B 'B 
BBB?B/.B?/
B<9B?;B<<B?Tzcua-driver-loop)r   r   r         @r   z)cua-driver asyncio bridge failed to startr   )	rf  is_aliverh  clearr   r   r  waitRuntimeError)rj  r   s   ` r    r  z_AsyncBridge.start8  s    < 	DL1133 	F
	 
	 
	 
	 
	 
	 !'tDGXYYY{,, 	LJKKK	L 	Lr"         >@r   Optional[float]r   c                :   ddl m} | j        r | j        r| j                                        s7t          j        |          r|                                 t          d           ||| j                  }|t          d          |	                    |          S )Nr   )safe_schedule_threadsafecua-driver bridge not startedr   )
agent.async_utilsr|  re  rf  ru  rn  iscoroutiners  rx  result)rj  coror   r|  futs        r    r   z_AsyncBridge.runN  s    >>>>>>z 	@ 	@T\5J5J5L5L 	@"4(( 

>???&&tTZ88;>???zz'z***r"   c                    | j         r=| j                                         r$| j                             | j         j                   | j        r| j                            d           d | _        d | _         d S )Ng       @r   )re  
is_runningcall_soon_threadsafestoprf  r   ri  s    r    r  z_AsyncBridge.stopY  sp    : 	=$*//11 	=J++DJO<<<< 	+Lc***


r"   Nr   ry  )r   rz  r   r   )__name__
__module____qualname____doc__rk  r  r   r  rZ   r"   r    rc  rc  0  ss        XX( ( ( (
L L L L,	+ 	+ 	+ 	+ 	+     r"   rc  c                     e Zd ZdZd+dZd,dZd,dZd-dZd,dZd,dZ	d,dZ
d,dZd,dZd.dZd/d0dZd1dZed2d            Zed3d            Zed4d!            Zed4d"            Zd,d#Zd5d&Z ed'd(h          Zd6d5d*ZdS )7_CuaDriverSessionu  Holds the mcp ClientSession. Spawned lazily; re-entered on drop.

    Lifecycle ownership: a single long-running coroutine
    (`_lifecycle_coro`) opens both the stdio_client and ClientSession
    contexts, populates capabilities, sets `_ready_event`, and then waits
    on `_shutdown_event`. When shutdown is signalled the same coroutine
    closes the contexts — keeping anyio's cancel-scope task-identity
    invariant intact (the bridge schedules each `bridge.run(coro)` as a
    NEW task, so opening contexts in one and closing them in another
    raises "Attempted to exit cancel scope in a different task").
    Tool calls run in their own short-lived tasks; they only touch the
    session object, never the surrounding contexts.
    bridgerc  r   r   c                    || _         d | _        t          j                    | _        d| _        i | _        d| _        t          j                    | _	        d | _
        d | _        d | _        d S )NFrz   )_bridge_sessionr   Lock_lock_started_capabilities_capability_versionrg  _ready_event_shutdown_event_lifecycle_future_setup_error)rj  r  s     r    rk  z_CuaDriverSession.__init__u  sf    ^%%
 .0(* %O--8<!%59r"   c                2    | j         st          d          d S )Nzcua-driver session not started)r  rx  ri  s    r    _require_startedz"_CuaDriverSession._require_started  s(    } 	A?@@@	A 	Ar"   c           	       K   ddl }ddlm}m} ddlm} ddlm} t          j	                    | _
        |                                }d| _        	 t                      }|st          t                                d| _        t!          |          \  }}	|                                }
 |||	 |t#                                          } ||          4 d{V \  }}d	| _         |||          4 d{V }|                                 d{V  |                                }d
| _        |                     |           d{V  || _        d| _        | j                                         t.                              d|                                |z
  |
|z
  ||
z
             | j
                                         d{V  ddd          d{V  n# 1 d{V swxY w Y   ddd          d{V  n# 1 d{V swxY w Y   n3# t4          $ r&}|| _        | j                                          d}~ww xY wd| _        d| _        dS # d| _        d| _        w xY w)u}  Long-lived owner of the stdio MCP contexts. Opens, signals
        ready, blocks on shutdown, then cleans up. enter + exit happen
        in the SAME asyncio task, so anyio's cancel-scope invariant
        holds — fixing the "Attempted to exit cancel scope in a
        different task than it was entered in" warning emitted by the
        previous _aenter/_aexit split.
        r   N)ClientSessionStdioServerParameters)stdio_clientr   zbinary-checkzmanifest-discovery)r   r   rj   zmcp-initializezcapability-discoveryreadyzBcua-driver session ready in %.1fs (manifest=%.1fs, mcp_init=%.1fs)F)timer1   r  r  mcp.client.stdior  r   r   rn  rg  r  	monotonic_startup_phaser   rx  r  r   rk   
initialize_populate_capabilitiesr  r  rq  r   r   rw  BaseExceptionr  r  )rj  _timer  r  r  r   _t0r   r   r   _t_manifestparamsrS   writesession_t_inites                    r    _lifecycle_coroz!_CuaDriverSession._lifecycle_coro  s      	<<<<<<<<111111EEEEEE  '}oo -A	"/11J >"#:#<#<===
 #7D3J??MGT//++K** -,-A-C-CDD  F $|F++ 6 6 6 6 6 6 6}e&6#(=u55 6 6 6 6 6 6 6!,,.........#oo//G +AD'55g>>>>>>>>>$+DM*1D'%))+++KK;))C/#c)+-   .33555555555+6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 66 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 60  	 	 	 !"D!!###	 !DM "DMMM !DM "DM!!!!st   BH G<8CGG<
G#	#G<&G#	'G<*H <
HH 	H
H I 
H>!H99H>>I I!r  r   c                  K   	 |                                  d{V }t          |dg           pg D ]}t          |dd          }t          |t                    s)t          |dd          }|(t          |dd          pi }|                    d          }t          |t
                    rd |D             | j        |<   t                      | j        |<   t          |dd          }|(t          |dd          pi }|                    d          }t          |t                    r	|| _        dS dS # t          $ r&}t                              d|           Y d}~dS d}~ww xY w)	u   Surface 4: cache per-tool capability sets + capability_version
        from tools/list. Soft prerequisite — discovery failure leaves
        the map empty and supports_capability degrades to False.Ntoolsr   capabilitiesmodel_extrac                <    h | ]}t          |t                    |S rZ   r   )rr   cs     r    rt   z;_CuaDriverSession._populate_capabilities.<locals>.<setcomp>  s7     5 5 5:a+=+=55 5 5r"   capability_versionz5cua-driver tools/list capability discovery failed: %s)
list_toolsgetattrr,   r   r   r   r  rq  r  rE   r   debug)	rj  r  
tools_listtool	tool_namecapsextracvr  s	            r    r  z(_CuaDriverSession._populate_capabilities  s     	U&1133333333J
GR88>B : :#D&$77	!)S11 t^T::< $D->>D"E 99^44DdD)) :5 5#'5 5 5D&y11 58EED&y11
 %94@@Bz
M4@@FBYY344"c"" .+-(((. . 	U 	U 	ULLPRSTTTTTTTTT	Us   D)D1 1
E!;EE!c                    | j         5  | j        r	 d d d            d S | j                                         |                                  d| _        d d d            d S # 1 swxY w Y   d S )NT)r  r  r  r  _start_lifecycle_lockedri  s    r    r  z_CuaDriverSession.start	  s    Z 	! 	!} 	! 	! 	! 	! 	! 	! 	! 	! L   ((*** DM	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	!s   	A4AA#&A#c                   t          j                    | _        d| _        d| _        | j        j        }|t          d          t          j	        | 
                                |          | _        | j                            d          sI|                                  t          | dd          }ddlm} t          d	| d
 |             d          | j        t          d| j                   | j        dS )z^Spawn the lifecycle owner and wait for it to reach ready.
        Caller must hold self._lock.Nr}  ry  r   r  unknownr   )display_hermes_homezEcua-driver session never reached ready (timeout 30s; stuck in phase: z.). Run `hermes computer-use doctor` and check z&/logs/agent.log for the phase timings.z!cua-driver session setup failed: )r   rg  r  r  r  r  re  rx  rn  run_coroutine_threadsafer  r  rw  _signal_shutdown_lockedr  hermes_constantsr  )rj  loopphaser  s       r    r  z)_CuaDriverSession._start_lifecycle_locked  s@    &O-- # |!<>???!(!A  ""D"
 "
  %%d%33 	((*** D"2I>>E<<<<<<Q#(Q Q '&((Q Q Q   (GD4EGG $% )(r"   c                    | j         5  | j        s	 d d d            d S d| _        |                                  d d d            d S # 1 swxY w Y   d S )NF)r  r  _stop_lifecycle_lockedri  s    r    r  z_CuaDriverSession.stop6  s    Z 	* 	*= 	* 	* 	* 	* 	* 	* 	* 	* "DM'')))		* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	*s   	AAA
A
c                \   |                                   | j        }|dS 	 |                    d           na# t          j        j        $ r t                              d           Y n1t          $ r%}t                              d|           Y d}~nd}~ww xY wd| _        dS # d| _        w xY w)zbSignal shutdown + wait for the lifecycle coroutine to unwind.
        Caller must hold self._lock.Nrt  r   z*cua-driver session shutdown timed out (5s)zcua-driver shutdown error: %s)	r  r  r  
concurrentfuturesTimeoutErrorr   warningrE   )rj  r  r  s      r    r  z(_CuaDriverSession._stop_lifecycle_locked=  s     	$$&&&$;F
	*JJsJ####!. 	I 	I 	INNGHHHHH 	? 	? 	? NN:A>>>>>>>>	?
 &*D"""TD"))))s8   8 B" .B&B" (	B1BB" BB" "	B+c                    | j         j        }| j        }|D|D|                                r2	 |                    |j                   dS # t          $ r Y dS w xY wdS dS dS )z8Set the asyncio shutdown event from the caller's thread.N)r  re  r  r  r  rq  rx  )rj  r  events      r    r  z)_CuaDriverSession._signal_shutdown_lockedP  s    |!$ 1doo6G6G 1))%)44444     1 1 1 1s   A	 	
AAr   r   r   r   c                f   K   | j                             ||           d {V }t          |          S r   )r  	call_tool_extract_tool_result)rj  r   r   r  s       r    _call_tool_asyncz"_CuaDriverSession._call_tool_async[  s=      }..tT::::::::#F+++r"   N
capabilityr  r   r   c                    |)| j                             |t                                v S t          fd| j                                         D                       S )u  Return True when the connected cua-driver advertises the given
        capability token (trycua/cua#1961 capability vocabulary).

        When ``tool`` is given, scope the check to that specific tool's
        advertised capability set. When omitted, return True if ANY tool
        advertises the capability — useful for "is this feature available
        anywhere on the driver" probes.

        Always returns False before the session is started (so consumers
        on a dead/uninitialised wrapper degrade rather than crash).
        Nc              3      K   | ]}|v V  	d S r   rZ   )rr   r  r  s     r    r   z8_CuaDriverSession.supports_capability.<locals>.<genexpr>n  s(      NN$:%NNNNNNr"   )r  r   rq  r   values)rj  r  r  s    ` r    supports_capabilityz%_CuaDriverSession.supports_capability`  s_     !3!7!7cee!D!DDDNNNN$2D2K2K2M2MNNNNNNr"   c                    || j         v S )um  Return True when ``tools/list`` advertised a tool by this name.

        Used to route capture(): cua-driver dropped the standalone
        ``screenshot`` tool and folded full-window PNG capture into
        ``get_window_state`` (whose own description notes it "Also captures
        a PNG screenshot of the specified window"). Older drivers that still
        expose ``screenshot`` keep using it; newer ones fall through to
        ``get_window_state``.

        Returns False when discovery hasn't populated the map yet — callers
        treat that as "unknown" and probe defensively rather than trusting it.
        )r  )rj  r   s     r    	_has_toolz_CuaDriverSession._has_toolp  s     t)))r"   c                *    t          | j                  S )zTrue once ``tools/list`` populated the per-tool map. When False,
        ``_has_tool`` answers are not trustworthy (discovery failed or the
        session hasn't started) and capture() should probe defensively.)r   r  ri  s    r    capabilities_discoveredz)_CuaDriverSession.capabilities_discovered  s    
 D&'''r"   c                    | j         S )u   Driver-advertised capability vocabulary version (empty string
        when the driver predates the field — older builds had no version).)r  ri  s    r    r  z$_CuaDriverSession.capability_version  s     ''r"   excrE   c                    | j         j        }t          | j         dd          }|dv p4|                    d          od|v pt	          | t
          t          f          S )zHReturn True for MCP/stdio failures that are recoverable by reconnecting.r  rz   >   EndOfStreamBrokenResourceErrorClosedResourceErroranyioResource)	__class__r  r  r   r,   BrokenPipeErrorEOFError)r  r   modules      r    _is_closed_session_errorz*_CuaDriverSession._is_closed_session_error  sf     }%b99QQ <!!'**AzT/A<#:;;	
r"   c                @    t          |           }d|v pd|v pd|v pd|v S )u  Return True for the cua-driver daemon-proxy EAGAIN congestion error.

        On macOS the ``cua-driver mcp`` bridge forwards calls to the CuaDriver
        daemon over a non-blocking unix socket. Heavier ops (notably
        ``get_window_state``, which walks the AX tree and captures a PNG) can
        come back as an ``McpError`` carrying ``Resource temporarily
        unavailable (os error 35)`` — POSIX EAGAIN — when the socket buffer is
        momentarily full. This is transient by definition: the same call
        succeeds when retried after a short pause (which is why spaced-out
        single calls work while rapid/large ones intermittently fail). Detect
        it by message so we can retry with backoff rather than surfacing an
        empty 0x0 capture to the model. See the EAGAIN diagnosis in
        references/catalog-add-troubleshooting (apple-music skill) and the
        cua-driver daemon-proxy note.
        z Resource temporarily unavailablezos error 35zdaemon transport errorzdaemon proxy)r   )r  r   s     r    _is_transient_daemon_errorz,_CuaDriverSession._is_transient_daemon_error  sE    " #hh.#5 %#%'3.% $		
r"   c                   | j         rH	 |                                  n2# t          $ r%}t                              d|           Y d}~nd}~ww xY wd| _         i | _        d| _        |                                  d| _         dS )zRecreate the MCP session after the daemon/stdin transport was closed.
        Caller must hold self._lock (the reconnect-once retry path holds it).z6cua-driver session cleanup before reconnect failed: %sNFrz   T)r  r  rE   r   r  r  r  r  rj  r  s     r    _restart_session_lockedz)_CuaDriverSession._restart_session_locked  s     = 	ZZ++---- Z Z ZUWXYYYYYYYYZ#% $$&&&s    
AAAr   r   c                   ddl }ddl}ddl}ddlm} t          |          }d}	|dk    r7d|vr3|                    dd          \  }
}	t          j        |
           |	|d<   t                      }|st          t                                |d	|t          j        |          g}d
}d}d}d}	 t          |          D ]h}	 |                    |ddt!          d|           |t#                                          }n(# t$          $ r}t          d| d|           |d}~ww xY w|j        pd                                }|dd         p|j        pddd         }t-          d |                    d          |                    d          fD             d          }|dk    r;	 t          j        ||d                   }n# t          j        $ r d}Y nw xY w||} nF||dz
  k     r;t4                              d||dz   ||           |                    |           |dz  }j|t          d| d| d|           g }d}t;          |t
                    r|nd}d}t;          |t
                    rj|                    d          du p|                    d          du }|                    d           }|s|                    d!          p|	}|rt          j                             |          r	 tC          |d"          5 }tE          j#        |$                                          %                    d#          }ddd           n# 1 swxY w Y   n3# t$          $ r&}t4          &                    d$||           Y d}~nd}~ww xY w|r|'                    |           |                    d%          }|)|                    d&          }|| d'nd} | r|  d(| n|}||||d)|	rEt          j                             |	          r'	 t          j(        |	           S # tR          $ r Y S w xY wS S # |	rEt          j                             |	          r'	 t          j(        |	           w # tR          $ r Y w w xY ww w xY w)*u  Fallback transport: invoke ``cua-driver call <tool> <json>`` as a
        subprocess instead of going through the stdio MCP bridge.

        The ``cua-driver mcp`` stdio bridge can persistently fail to forward
        heavier calls (notably ``get_window_state``) to the daemon with POSIX
        EAGAIN, while the plain ``cua-driver call`` path — which talks to the
        daemon over its own socket — keeps working. When the MCP path gives up,
        we retry over the CLI and remap the JSON into the same dict shape that
        ``_extract_tool_result`` produces, so callers (capture(), _action(),
        list_windows parsing) are transport-agnostic.

        For ``get_window_state`` we route the screenshot to a temp file via
        ``screenshot_out_file`` so the daemon returns a tiny JSON body (a path)
        instead of a multi-megabyte base64 blob — the large payload is what
        congests the daemon socket and triggers EAGAIN in the first place. We
        read the PNG back from disk and base64-encode it ourselves. The CLI
        call is itself retried a few times with backoff, since the underlying
        daemon socket can still be momentarily busy.
        r   Nr   get_window_statescreenshot_out_file	cua_shot_z.png)prefixsuffixcallr
  g      ?rz   Tg      .@)r   r   r   rj   zcua-driver CLI fallback for z failed to spawn:    c              3  &   K   | ]}|d k    |V  dS )NrZ   )rr   rB  s     r    r   z7_CuaDriverSession._call_tool_via_cli.<locals>.<genexpr>  s&      JJ1!r''Q''''JJr"   {[r  )defaultru   zMcua-driver CLI fallback for %s got no JSON (attempt %d/%d); retrying in %.1fsr   z returned no JSON after z attempts: FisErroris_errorscreenshot_png_b64screenshot_file_pathrbasciiz.cua-driver CLI fallback: failed reading %s: %stree_markdownelement_countz	 elementsrJ  )r   imagesstructuredContentr  )*r   tempfiler  r   r   r-   mkstemprP   rs  r   rx  r  r   dumpsranger   maxrk   rE   rx   r   r   minfindr   JSONDecodeErrorr   r  sleepr,   r   r&   existsrR   base64	b64encoderS   decoder  r  removeOSError)!rj  r   r   r   _subprocess	_tempfiler  r   	call_args	shot_filefdr   rU  attemptsbackoffparsedlast_errattemptr   r  r   r  r   r	  r   r   r  shotfpathfhrN  ecrM  s!                                    r    _call_tool_via_cliz$_CuaDriverSession._call_tool_via_cli  s   ( 	)((($$$$EEEEEEJJ	#'	%%%*?y*P*P%--[-PPMB	HRLLL/8I+,+--
 	:68899964I)>)>?N	 ?? ! !j&??DtSwEWEW445I5K5KLL +  DD ! j j j&'ad'a'a^_'a'abbhiij {(b//11tt9A):DSD(AJJ# >JJJ   B;;)$(Js566{$;$;		/ ) ) )$(			) ,!*X\))NN=gk8W  
 KK(((qLG~"74 7 77 7,47 7   !#FD3=fd3K3K)UQUJH&$'' E "::i00D8ZFJJz<R<RVZ<Zzz"677 e"JJ'=>>K)E e!6!6 ee!%eT!2!2 Sb'-'7		'B'B'I'I''R'RS S S S S S S S S S S S S S S( e e e"LL)Y[`bcdddddddde (MM$'''zz/22#O44B24.....bG3:Dg/////D %/#	   RW^^I66 Ii((((   D y RW^^I66 Ii((((   D s   -P.  =C>=P. >
D#DD##BP. &GP. GP. GDP. 3M :M	=M 	MM MM P. 
NN ;P.  NA P. P
P)(P)."Q8Q&%Q8&
Q30Q82Q33Q8start_sessionend_sessionry  c                   | j         s8|| j        vr/t                              d|           |                                  |                                  	 | j                            |                     ||          |          S # t          $ r}| 
                    |          r8t                              d||           |                     |||          cY d }~S |                     |          s t                              d|           | j        5  |                                  d d d            n# 1 swxY w Y   | j                            |                     ||          |          cY d }~S d }~ww xY w)Nz=cua-driver session not active on %s; (re)starting before callr   zIcua-driver MCP transport failed on %s (%s); falling back to CLI transportz:cua-driver MCP session closed during %s; reconnecting once)r  _LIFECYCLE_CALLSr   r  r  r  r  r   r  rE   r  r(  r  r  r  )rj  r   r   r   r  s        r    r  z_CuaDriverSession.call_tool<  s    } 	T-B!B!BNNOQU   JJLLL	X<##D$9$9$$E$Ew#WWW 	X 	X 	X..q11 D4591   ..tT7CCCCCCCC0033  NNWY]^^^ / /,,.../ / / / / / / / / / / / / / /<##D$9$9$$E$Ew#WWWWWWWW	XsO   /B 
E:AE5E:8E5D5)E55D9	9E5<D9	=2E5/E:5E:)r  rc  r   r   r   )r  r   r   r   r   r   r   r   r   r   r   )r  r   r  r   r   r   )r   r   r   r   r   r   r   r   )r  rE   r   r   )r   r   r   r   r   r   r   r   r  )r  r  r  r  rk  r  r  r  r  r  r  r  r  r  r  r  propertyr  r  staticmethodr  r  r  r(  	frozensetr,  r  rZ   r"   r    r  r  f  s        : : : :(A A A AW" W" W" W"r!U !U !U !UF! ! ! !#% #% #% #%J* * * ** * * *&	 	 	 	, , , ,
O O O O O * * * * ( ( ( X( ( ( ( X(
 
 
 
 \
 
 
 
 \
0   v v v vv !y/=!ABB'X 'X 'X 'X 'X 'X 'Xr"   r  
mcp_resultr   c                   d}g }g }t          | dd          du }t          | dd          pd}g }t          | dg           pg D ]}t          |dd          }|dk    r'|                    t          |dd	          pd	           @|d
k    rPt          |dd          }	|	r=|                    |	           t          |dd          pd	}
|                    |
           |rud                    d |D                       }	 |                                                    d          rt          j        |          n|}n# t
          j        $ r |}Y nw xY w|||||dS )u  Convert an mcp CallToolResult into a plain dict.

    cua-driver returns a mix of text parts, image parts, and structuredContent.
    We flatten into:
      {
        "data": <text or parsed json>,
        "images": [b64, ...],
        "image_mime_types": [mime, ...],   # parallel to `images`, "" when absent
        "structuredContent": <dict|None>,
        "isError": bool,
      }
    structuredContent is populated from the MCP result's structuredContent field
    (MCP spec §2024-11-05+) and takes precedence for structured data like
    list_windows window arrays.

    `image_mime_types` is the explicit `mimeType` cua-driver emits on every
    image part as of trycua/cua#1961 (Surface 7 of
    NousResearch/hermes-agent#47072). Each entry corresponds index-for-index
    with `images`; an empty string entry signals the part carried no
    mimeType (older cua-driver build), and the caller should fall back to
    base64-prefix sniffing.
    Nr  FTr
  contenttyper   rz   imager   mimeTyperJ  c              3     K   | ]}||V  	d S r   rZ   )rr   ts     r    r   z'_extract_tool_result.<locals>.<genexpr>  s'      77Q71777777r"   )r  r  )r   r	  image_mime_typesr
  r  )r  r  r   r   r   r   r   r  )r3  r   r	  r;  r  r   text_chunksr_  ptypeb64mimejoineds               r    r  r  f  s   . DF"$ z9e44<H!(5H$!O!O!WSWJK
Ir228b 	. 	.fd++F??wtVR88>B????g$--C .c"""tZ66<" ''--- 77k77777	)/)B)B:)N)NZ4:f%%%TZDD# 	 	 	DDD	 ,'  s   ;=D9 9EEr   #tuple[Optional[str], Optional[str]]c                   |                      d          pg }|r=|d         r5|                      d          pg }|r|d         r|d         nd}|d         |fS |                      d          pi }|                     d          p|                     d          }|r0|                     d          p|                     d	          pd}||fS d
S )u  Pull a (png_b64, mime_type) pair out of a flattened tool result.

    cua-driver delivers window screenshots in two shapes depending on tool +
    transport:

      * As an MCP ``image`` content part — surfaced by ``_extract_tool_result``
        in ``out["images"]`` with a parallel ``image_mime_types`` entry. This
        is what ``get_window_state`` emits over the stdio MCP transport.
      * As a base64 field inside ``structuredContent`` —
        ``screenshot_png_b64`` (+ ``screenshot_mime_type``). This is what
        ``get_window_state`` returns when its structured payload carries the
        image instead of a content part (newer driver builds; also the shape
        seen via the ``cua-driver call`` CLI surface).

    Checking both makes capture() robust to either delivery shape, so the
    image never silently drops just because the driver moved it between the
    content list and structuredContent. Returns ``(None, None)`` when neither
    location carries an image.
    r	  r   r;  Nr
  r  png_b64screenshot_mime_type	mime_type)NNr   )r   r	  mimesr?  r   r>  s         r    _image_from_tool_resultrG    s    ( WWX$"F &) *++1r 7U1X7uQxx4ay$,--3J
..-
.
.
K*..2K2KC
 NN122 ~~k** 	
 Dy:r"   c                    t          | t                    st          | t          t          f          sdS 	 t          |           }n# t          $ r Y dS w xY w|dk    r|ndS )zCReturn a positive integer, rejecting booleans and malformed values.Nr   )r,   r   r_   r   ra   )r   r!  s     r    _positive_intrI    st    % jc
&C&C tU   ttaZZ66T)s   A 
AAraw_windowsc                   g }| D ]}t          |                    d                    }t          |                    d                    }||K|                    d          }t          |t          t          f          rt          |t
                    s|nd}|                    |                    dd          |||                    d          d	u |                    d
d          |d           |S )u   Normalise cua-driver ``list_windows`` entries, dropping unusable ones.

    Every downstream operation needs both an integer ``pid`` (for
    get_window_state / action tools) and ``window_id`` (for screenshot /
    element clicks), so a window missing either is uncapturable.

    Crucially, on X11 a window's PID comes from the *optional*
    ``_NET_WM_PID`` property — the desktop root, panels, and
    override-redirect popups routinely omit it, so the driver reports
    ``pid: null`` for them. Coercing every entry unconditionally
    (``int(w["pid"])``) let one such window abort enumeration of the real,
    targetable windows. We skip the unusable entries instead so capture()
    and focus_app() still find the windows that matter.

    ``z_index`` follows CUA Driver semantics: higher = closer to front.
    Wayland may return ``z_index: null`` (undefined stacking order); we
    treat null as the lowest priority so real windows still sort above
    desktop/root windows, and the backmost never ends up selected as the
    capture target.
    pidr   Nrp   r   app_namerz   is_on_screenFr   rM  rL  r   r   r   rp   )rI  r   r,   r_   r   r   r  )rJ  rl   rs   pid_intwindow_id_intz_rawrp   s          r    _ingest_windowsrS    s    * %'G  e--%aeeK&8&899?m3i  %ec5\::a:eUYCZCZa%%`aj"--& %%//58UU7B''	
 	
 		 		 		 		 Nr"   c            	         e Zd ZdZddZddZddZddZdd	ZdddZ	ddZ
ddZddZ	 	 	 	 dddZdd$Zdddd%d&ddd'd(dd2Zddddd%ddd'd3dd9Zd:dddddd'd;dd>Zdd'd?ddAZdd'd?ddCZdddEZddFZddGZdddIZddddd'dJddOZddPZddQddRZddSZddUZddVZdWdXdYdZddaZddbddeZdddddfddkZ dddddlddpZ!ddbddqZ"d'drdduZ#ddvZ$ddwZ%d'dWdxdd|Z&dd}Z'dd~Z(ddZ)ddZ*ddZ+dddddZ,ddZ-ddZ.dS )CuaDriverBackendz@Default computer-use backend. Cross-platform via cua-driver MCP.r   r   c                    t                      | _        t          | j                  | _        d | _        d | _        d | _        d | _        i | _        dt          j
                    j        d d          | _        d S )Nzhermes-   )rc  r  r  r  _active_pid_active_window_id	_last_app_last_target_snapshot_tokensuuiduuid4hex_session_idri  s    r    rk  zCuaDriverBackend.__init__  sv    #~~)$,77*.04(, AE 13$ !B$*,,*:3B3*? A Ar"   c                   t                       ddlm}  |dd           dd l}|                                 | j                                         	 | j                            dd| j        i           n2# t          $ r%}t                              d|           Y d }~nd }~ww xY w| j        j        rt                      }|rJ	 |                     |	           n2# t          $ r%}t                              d
|           Y d }~nd }~ww xY wt                      rT	 |                     d| j                   d S # t          $ r&}t                              d|           Y d }~d S d }~ww xY wd S d S )Nr   )ensureztool.computer_useF)promptr)  r  z:cua-driver start_session failed (continuing anonymous): %s)r^   z5cua-driver set_config(max_image_dimension) failed: %s	cursor_idz.cua-driver set_agent_cursor_enabled failed: %s)r  tools.lazy_depsrb  	importlibinvalidate_cachesr  r  r  r`  rE   r   r  r  rc   
set_configrW   set_agent_cursor_enabled)rj  _lazy_ensurerg  r  max_dims        r    r  zCuaDriverBackend.start'  s    	;:::::(7777 	##%%%	ZM##OiAQ5RSSSS 	Z 	Z 	ZLLUWXYYYYYYYY	Z =! 	V 899G ]]OOO@@@@  ] ] ]LL!XZ[\\\\\\\\]
    VV11%4CS1TTTTT  V V VLL!QSTUUUUUUUUUV	V 	VV VsH   "A7 7
B&B!!B&C 
D'DDD< <
E,E''E,c                l   | j         j        rV	 | j                             dd| j        i           n2# t          $ r%}t
                              d|           Y d }~nd }~ww xY w	 | j                                          | j                                         d S # | j                                         w xY w)Nr*  r  z7cua-driver end_session failed (continuing teardown): %s)	r  r  r  r`  rE   r   r  r  r  r  s     r    r  zCuaDriverBackend.stop\  s     =! 	[[''	4CS7TUUUU [ [ [VXYZZZZZZZZ[	 M   LDLs    "1 
A AA $B B3r   c                >    t           j        dvrdS t                      S )N)rI   r   rJ   F)rN   rO   r   ri  s    r    is_availablezCuaDriverBackend.is_availablel  s#    
 <;;;5*,,,r"   c                L    d| _         d| _        d| _        d| _        i | _        dS )zGForget a capture/focus target so a failed lookup cannot misroute input.N)rX  rY  rZ  r[  r\  ri  s    r    _clear_active_targetz%CuaDriverBackend._clear_active_targetu  s/    !%  "r"   rz   moder   r   r
   c           
     X    |                                   t          |dddg d|d          S )zAReturn an empty capture after disarming any prior target context.r   Nrz   )rr  r@  rA  rC  r  appwindow_titlepng_bytes_len)rq  r
   )rj  rr  r   s      r    _failed_capturez CuaDriverBackend._failed_capture}  sC    !!### 	
 	
 	
 		
r"   r   r   r   c                j   	 | j                             ||          }n## t          $ r |                                   w xY w|                    d          du r[|                    d          }|                                  t          d| dt          |t                    r|rd| ndz             |S )zKCall a capture-stage tool and disarm state on transport or logical failure.r  Tr   r   z failedz: rz   )r  r  rE   rq  r   rx  r,   r   )rj  r   r   r   r   s        r    _call_capture_toolz#CuaDriverBackend._call_capture_tool  s    	-))$55CC 	 	 	%%'''	 779%%ggfooG%%'''+d+++%/%=%=Q'Q>>>>rS   
s     >rm   c                   |                      dd| j        d          }|                    d          pi                     d          pg }t          |          }|                    d d           |r|S t
                              d           	 | j                            dd| j        dd	          }n4# t          $ r'}t
          
                    d
|           g cY d}~S d}~ww xY w|                    d          du r0t
          
                    d           |                                  g S |                    d          pi                     d          pg }t          |          }|                    d d           |S )u  Load normalized visible windows, with the shared CLI recovery path.

        Windows are sorted by ``z_index`` **descending**: CUA Driver
        defines higher values as closer to the front, so the frontmost
        window ends up at index 0 — which is what ``capture()`` and
        ``focus_app()`` pick as the default target.  ``_ingest_windows``
        already normalised null ``z_index`` (Wayland) to 0, so those
        windows sort to the back.
        list_windowsT)on_screen_onlyr  r
  rl   c                    | d         S Nrp   rZ   rs   s    r    <lambda>z0CuaDriverBackend._load_windows.<locals>.<lambda>  
    1Y< r"   )r   reversezScua-driver list_windows returned no windows over MCP; re-fetching via CLI transportg      4@z3cua-driver CLI re-fetch for list_windows failed: %sNr  z:cua-driver CLI re-fetch for list_windows returned an errorc                    | d         S r~  rZ   r  s    r    r  z0CuaDriverBackend._load_windows.<locals>.<lambda>  r  r"   )ry  r`  r   rS  sortr   r  r  r(  rE   r   rq  )rj  r   rJ  rl   cli_outr  s         r    _load_windowszCuaDriverBackend._load_windows  s    %%#0@AA
 
 ww2339r>>yIIOR!+..//>>> 	N,	
 	
 	
	m66#'D4DEE GG
  	 	 	LLNPSTTTIIIIII	 ;;y!!T))LLUVVV%%'''I{{#677=2BB9MMSQS!+..//>>>s   $B6 6
C' C"C'"C'rl   rt  c                   |                                                                 sg S fd|D             }|r|S 	 |                                 }n5# t          $ r(}t                              d||           g }Y d}~nd}~ww xY wt                      t                      |D ]t          t                    r	                    d          du r/	                    d          }t          |t                    st          |t          t          f          sv	 t          |          }n# t          $ r Y w xY w|dk    rfdd	D             }|v r                    |           t          fd
|D                       r                    |           fd|D             }	|	r|	S fd|D             }
|
r|
S fd|D             }|r|S fd|D             S )a  Resolve ``app=`` through exact names before convenience substrings.

        Linux ``list_windows`` can omit an app name while ``list_apps`` retains
        name/bundle-ID metadata. Exact direct names and exact metadata aliases
        must win over substring matches: querying ``Code`` must not silently
        select ``Visual Studio Code`` merely because it is frontmost.
        c           	         g | ]O}t          |                    d d                                                                                    k    M|PS rM  rz   r   r   r   rT   rr   rs   	app_lowers     r    r   z;CuaDriverBackend._match_windows_for_app.<locals>.<listcomp>  s\     
 
 
Cj" 5 566<<>>DDFFFF FFFr"   z1computer_use list_apps fallback failed for %r: %sNrunningFrL  r   c                    h | ]f}t                              |          xt                    ,                                @                                                                gS rZ   )r,   r   r   r   rT   )rr   r   raw_appr   s     r    rt   z:CuaDriverBackend._match_windows_for_app.<locals>.<setcomp>  sr       C(8(88u3?? EJKKMM##%%  r"   )	bundle_idbundleIdr   rM  display_namec              3      K   | ]}|v V  	d S r   rZ   )rr   aliasr  s     r    r   z:CuaDriverBackend._match_windows_for_app.<locals>.<genexpr>  s(      ==EY%'======r"   c                B    g | ]}|                     d           v |S rL  r   )rr   rs   
exact_pidss     r    r   z;CuaDriverBackend._match_windows_for_app.<locals>.<listcomp>  s-    KKKe
0J0J!0J0J0Jr"   c           	         g | ];}t          |                    d d                                                    v 9|<S r  )r   r   rT   r  s     r    r   z;CuaDriverBackend._match_windows_for_app.<locals>.<listcomp>   sP     
 
 
Cj" 5 566<<>>>> >>>r"   c                B    g | ]}|                     d           v |S r  r   )rr   rs   partial_pidss     r    r   z;CuaDriverBackend._match_windows_for_app.<locals>.<listcomp>  s-    OOO!!%%,,,2N2NA2N2N2Nr"   c           	         g | ]p}t          |                    d d                                                    s9t          |                    dd                                                    v n|qS )rM  rz   r   r  r  s     r    r   z;CuaDriverBackend._match_windows_for_app.<locals>.<listcomp>  s|     
 
 
quuZ,,--3355
 Sw!3!344::<<<< <<<r"   )r   rT   	list_appsrE   r   r  rq  r,   r-   r   r   r_   r   ra   addr   )rj  rl   rt  direct_exactrunning_appsr  raw_pidrL  aliasesmetadata_exactdirect_partialmetadata_partialr  r  r  r  r   s               @@@@@r    _match_windows_for_appz'CuaDriverBackend._match_windows_for_app  s    IIKK%%''	 	I
 
 
 

 
 
  	 	>>++LL 	 	 	 LLLcSVWWWLLLLLL	  #uu
!$# 	& 	&Ggt,, I0F0F%0O0Okk%((G'4(( 
7S#J0O0O 'll   axx    X  G
 G##s####====W===== &  %%%KKKKWKKK 	"!!
 
 
 

 
 
  	"!!OOOOwOOO 	$##

 
 
 

 
 
 	
s*   A 
B
"BB
"D22
D?>D?somNr   rL  r\   r   c                  # ||_|||                      |d          S t          |          }t          |          }|||                      |d          S |pd||ddddg}nP	 |                                 }n## t          $ r |                                   w xY w|s|                      |          S |r|p|rn|                                                                t          v rAd6d##fd|D             }|s|                      |d|d          S t          |d           }n:|8|6|r4| 	                    ||          }	|	s|                      |d|d          S |	}t          |t          |          |dup|du          }
|
d         | _        |
d         | _        i | _        |
d         }|s| j        s|p|pd| _        | j        | j        d| _        d}d}g }dx}}d}|dk    r| j                            d          p| j        j         }d}|r;|                     d| j        dd| j        d          }t-          |          \  }}|sd}||                     d| j        | j        | j        d          }t-          |          \  }}t/          |d          t0                    r|d          nd}t3          |          \  }}t5          j        d!|          }|r|                    d"          }|st:                              d#| j                   	 | j                            d| j        | j        | j        dd$          }|                     d%          d&u r|                                  n%|                     d'          r|d'         d         }d(}n.# t          $ r&}t:          !                    d)|           Y d}~nd}~ww xY wn|                     d| j        | j        | j        d          }d7d+} ||          rt:                              d,| j        | j                   	 | j                            d| j        | j        | j        dd$          }|                     d%          d&u r|                                  n ||          s|}n2# t          $ r%}t:          !                    d-|           Y d}~nd}~ww xY wt/          |d          t0                    r|d          nd}t3          |          \  }}t5          j        d.|          }|                     d/          pi                      d0          }t/          |tD                    r|rtG          |          }n|rtI          |          ng }d1 |D             | _        t-          |          \  }}t5          j        d!|          }|r|                    d"          }d}|rf	 tK          j&        |d2          } tO          |           }tQ          |           \  }!}"|!r|"r|!}|"}n%# t          $ r tO          |          d3z  d4z  }Y nw xY wtS          |||||||||5	  	        S )8u   Capture the frontmost on-screen window or an exact known target.

        Maps hermes `capture(mode, app)` → cua-driver `list_windows` +
        `get_window_state` (ax/som) or `screenshot` (vision).
        Nz3<capture targeting requires both pid and window_id>z?<capture targeting requires positive integer pid and window_id>rz   Fr   rO  rs   r   r   r   c                    |                      dd           d|                      dd                                           t          fdt          D                       S )NrM  rz    r   c              3      K   | ]}|v V  	d S r   rZ   )rr   r   haystacks     r    r   zGCuaDriverBackend.capture.<locals>._is_desktop_window.<locals>.<genexpr>V  s(      NN48+NNNNNNr"   )r   rT   r   _DESKTOP_WINDOW_NAMES)rs   r  s    @r    _is_desktop_windowz4CuaDriverBackend.capture.<locals>._is_desktop_windowT  sb    eeJ33JJaeeGR6H6HJJPPRRNNNN8MNNNNNNr"   c                *    g | ]} |          |S rZ   rZ   )rr   rs   r  s     r    r   z,CuaDriverBackend.capture.<locals>.<listcomp>X  s*    CCCQ-?-?-B-BCqCCCr"   z'<no desktop/shell window found for app=z; cua-driver captures one window at a time and exposes no whole-virtual-desktop or per-monitor capture. Call list_apps / capture(app='<AppName>') to target a specific window instead. On Windows the taskbar is 'Shell_TrayWnd' and the desktop is 'Progman'.>c                B     t           fddD                       rdndS )Nc              3     K   | ]G}|                     d d           d                     dd                                           v V  HdS )rM  rz   r  r   N)r   rT   )rr   rC  rs   s     r    r   z=CuaDriverBackend.capture.<locals>.<lambda>.<locals>.<genexpr>j  sr       ' ' AEE*b11HHAEE'24F4FHHNNPPP' ' ' ' ' 'r"   )r7   r8   r9   r<   r5   r   ru   )r   r  s   `r    r  z*CuaDriverBackend.capture.<locals>.<lambda>j  sH    3 ' ' ' '[' ' ' $ $ aa  r"   )r   z!<no on-screen window matched app=u   ; call list_apps to see available app names or bundle IDs (macOS reports localized names, e.g. '計算機' instead of 'Calculator'; some Linux/Qt apps only resolve via list_apps metadata)>)r   r   rL  r   rM  rL  r   vision
screenshotjpegU   )r   formatqualityr  r  )rL  r   r  r   zAXWindow\s+"([^"]+)"ru   zbcua-driver vision capture returned no image over MCP (window_id=%s); re-fetching via CLI transportry  r  Tr	  z	image/pngz8cua-driver CLI re-fetch for vision screenshot failed: %sr   c                   |                      d          rdS |                      d          pi }|                     d          s|                     d          rdS t          |                      d          t                    r|                      d          nd}t          |pd          \  }}|o|                                 S )Nr	  Fr
  r  r  r   rz   )r   r,   r   rO  r   )r   sc_txt_trs        r    _gws_is_emptyz/CuaDriverBackend.capture.<locals>._gws_is_empty  s    778$$ ! 5gg1228b 77:&& !#''2F*G*G ! 5)3CGGFOOS)I)IQcggfooor(332-288::..r"   zrcua-driver get_window_state returned an empty result over MCP (pid=%s window_id=%s); re-fetching via CLI transportz7cua-driver CLI re-fetch for get_window_state failed: %sz(\d+)\s+elements?r
  r  c                6    i | ]}|j         	|j        |j         S rZ   )r  r  )rr   r  s     r    
<dictcomp>z,CuaDriverBackend.capture.<locals>.<dictcomp>?  s6     % % %?%% % %r"   )validater	  r
  )	rr  r@  rA  rC  r  rt  ru  rv  image_mime_typers   r   r   r   )r   r   r   r   )*rw  rI  r  rE   rq  r   rT   _SCREEN_CAPTURE_SENTINELSsortedr  r   r   rX  rY  r\  rZ  r[  r  r  r  ry  r`  rG  r,   r   rO  r|   r}   r~   r   r  r(  r   r   r   r$  r  r  	b64decoderv   rF  r
   )$rj  rr  rt  rL  r   
target_pidtarget_window_idrl   r5   filteredr   rM  rC  r  r  r@  rA  ru  use_screenshotsc_outgws_outr   r  rN  wtr  cli_excr  rM  r  sc_elementsrv  r   detected_widthdetected_heightr  s$                                      @r    capturezCuaDriverBackend.capture  s   , ?i3{i/++O   's++J,Y77!%5%=++[    I2!-#  GG,,..   ))+++  2++D111 ;9,,9J9J9L9LPi9i9iO O O O DCCC'CCCG ++J# J J J
 
 
    GG [Y.3.227C@@H 
++<C < < <	 	 	 G (s))DAIT,A
 
 

 "%=!'!4 !#*%  	3dn 	3%22DN#/
 
 "&)-$&8 ''55 =}<<  04F "00 %)%;"(#%#'#3	   ,C6+J+J( " "F~11&#/%)%;#'#3   ,C7+K+K( +5WV_c*J*JRwvPR*4004Y6== /#%88A;;L  D*  
"m>>*#'#3)-)?'+'7 
  G {{9--55113333 X.. 6")("3A"6*5    LLRT[       1< --"+!%!7#/  G"/ / / / }W%% K$d&<  
"m>>*#'#3)-)?'+'7 
  G {{9--55113333*]733 *")    LLQSZ       
 '1#&F&FN76??BD,T22MGT 	.88A #;;':;;ArFFzRRK+t,, K K:;GG>BJ4T:::% %!% % %D! (?w'G'G$G_ 2D99B +!xx{{ 		66&w??? #C2Ns2S2S/! -o -*E,F 6 6 6 #Gq 0A 56 %'+

 

 

 
	
sP   &A; ; BB N 
O!OO+A(R 
SR>>S
?X
 
X,+X,r*   r+   bring_to_frontOptional[ActionResult]c                    |r|dk    rdS |dk    rt          d|dd|d          S | j                            d	|
          st          d|ddd          S d|d<   |rd|d<   dS )a8  Attach delivery_mode to an input-action args dict.

        Background is the default and never needs a flag. Foreground is only
        sent when the driver advertises support for it; on an older driver
        that lacks the capability we refuse with a structured
        ``foreground_unsupported`` result instead of silently downgrading to
        background (which would land the input somewhere the model didn't
        expect). Returns an ActionResult to short-circuit on refusal, or None
        to proceed. See NousResearch/hermes-agent#67052 phase B.
        
backgroundN
foregroundFbad_delivery_modezunknown delivery_mode u    — use background|foreground.)r   r*   r(   r   zinput.delivery_moder  foreground_unsupportedzThis cua-driver build does not support foreground delivery (no `input.delivery_mode` capability). Update cua-driver to escalate to the foreground rung.)r   r*   r(   r+   r   r+   Tr  )r	   r  r  )rj  r*   r   r+   r  s        r    _apply_deliveryz CuaDriverBackend._apply_deliveryh  s    "  	 = =4L((.Aaaaa   
 }00! 1 
 
 	  .F*E	    !-_ 	*%)D!"tr"   leftru   F)elementr  r  buttonclick_countr^  r+   r  r  r  r  r  r  r_   r^  Optional[List[str]]r	   c                  | j         }	|	t          ddd          S |pd                                }
|
dvrt          ddd|d          S |d	k    rd
nd}|	|
d}|)| j        t          d|d          S ||d<   | j        |d<   nD|0|.| j        t          d|d          S ||d<   ||d<   | j        |d<   nt          d|d          S |r||d<   |                     ||||          }||S |                     ||          S )NFclick*   No active window — call capture() first.r   r*   r   r  >   r  rightmiddlezunknown button u"    — expected left, right, middle.r   double_click)rL  r  z,No active window_id for element_index click.r  r   z)No active window_id for coordinate click.r  r  zclick requires element= or x/y.modifier)rX  r	   rT   rY  r  _action)rj  r  r  r  r  r  r^  r+   r  rL  button_normr  r   refusals                 r    r  zCuaDriverBackend.click  s    ;5(TV V V V '..009995(f&(f(f(fh h h h!,!1!1~~w'*kBB%-#uT,Z\ \ \ \$+D! $ 6D]q}%-#uT,WY Y Y YDIDI $ 6D5(IK K K K 	)(D&&tT=.QQN||D$'''r"   )from_element
to_elementfrom_xyto_xyr  r^  r+   r  r  r  r  Optional[Tuple[int, int]]r  c               @   | j         }	|	t          ddd          S d|	i}
|0|.| j        t          ddd          S ||
d<   ||
d<   | j        |
d	<   n||| j        t          ddd
          S t          |d                   t          |d                   c|
d<   |
d<   t          |d                   t          |d                   c|
d<   |
d<   | j        |
d	<   nt          ddd          S |                     d|
||          }||S |                     d|
          S )NFdragr  r  rL  z+No active window_id for element-based drag.r  r  r   z(No active window_id for coordinate drag.r   ru   from_xfrom_yto_xto_yzGdrag requires from_element/to_element or from_coordinate/to_coordinate.)rX  r	   rY  r_   r  r  )rj  r  r  r  r  r  r^  r+   r  rL  r   r  s               r    r  zCuaDriverBackend.drag  s    ;5(TV V V V %s|#
(>%-#uV,Y[ [ [ [#/D !+D $ 6D U%6%-#uV,VX X X X-0__c'!*oo*DNDN),U1XE!H&DL$v, $ 6D5(qs s s s&&vt]NSSN||FD)))r"   r	  )amountr  r  r  r^  r+   r  	directionr  c          	        | j         }	|	t          ddd          S |	|t          dt          d|                    d}
|| j        ||
d<   | j        |
d	<   nM|K|I| j        t          ddd
          S | j                            dd          r
||
d<   ||
d<   | j        |
d	<   |                     d|
||          }||S |                     d|
          S )NFscrollr  r  ru   2   )rL  r  r  r  r   z*No active window_id for coordinate scroll.zinput.scroll.coordinatesr  r  r  )	rX  r	   r  r  rY  r  r  r  r  )rj  r  r  r  r  r  r^  r+   r  rL  r   r  s               r    r  zCuaDriverBackend.scroll  s9    ;5(TV V V V "!SV__-- 
  

 4#9#E$+D! $ 6D]q}%-#uX,XZ Z Z Z }00* 1    S	S	 $ 6D&&x}nUUN||Hd+++r"   )r+   r  r   c                   | j         }| j        }||t          ddd          S |||d}|                     d|||          }||S |                     d|          S )NF	type_textr  r  )rL  r   r   )rX  rY  r	   r  r  )rj  r   r+   r  rL  r   r   r  s           r    r  zCuaDriverBackend.type_text	  s    *	;)+5(TV V V V'*DQQ&&{D-XXN||K...r"   rP  c                  | j         }| j        }||t          ddd          S t          |          \  }}|st          ddd| d          S |r<||||gz   d}|                     d|||          }	|	|	S |                     d|          S |||d	}|                     d
|||          }	|	|	S |                     d
|          S )NFr   r  r  zCould not parse key from ''.)rL  r   rP  hotkey)rL  r   r   	press_key)rX  rY  r	   ra  r  r  )
rj  rP  r+   r  rL  r   key_namer^  r   r  s
             r    r   zCuaDriverBackend.key,	  s.   *	;)+5(TV V V V /t44) 	O5(MT(M(M(MO O O O  	3+.Y,5
,B$D $DD**8T=.YYG"<<$///YxHHD**;m^\\G"<<T222r"   r   c                    | j         }| j        }||t          ddd          S |t          ddd          S ||||d}|                     d|          S )zBSet a value on an element. Handles AXPopUpButton selects natively.NF	set_valuer  r  z,set_value requires element= (element index).)rL  r   r  r   )rX  rY  r	   r  )rj  r   r  rL  r   r   s         r    r  zCuaDriverBackend.set_valueI	  s    *	;)+5(TV V V V?5(VX X X X "$	 
  
 ||K...r"   c           	     (   | j                             dd| j        i          }|                    d          }t	          |t
                    r0t	          |                    d          t                    r|d         S |                    d          }t	          |t                    r|S t	          |t
                    r0t	          |                    d          t                    r|d         S t	          |t                    rg }|                                D ]v}t          j
        d|          }|r]|                    |                    d                                          t          |                    d                    d	           w|S g S )
Nr  r  r
  appsr   z(.+?)\s+\(pid\s+(\d+)\)ru   r   )r   rL  )r  r  r`  r   r,   r-   r   r   
splitlinesr|   r}   r  r~   r   r_   )rj  r   r   r   r  liner  s          r    r  zCuaDriverBackend.list_apps\	  sg   m%%kIt?O3PQQWW011
j$'' 	&Jz~~f7M7Mt,T,T 	&f%% wwvdD!! 	KdD!! 	 j&1A1A4&H&H 	 <dC   	D)) V VI8$?? VKK)9)9););CPQ

OO T TUUUK	r"   c                *    |                                  S r   )r  ri  s    r    r{  zCuaDriverBackend.list_windowsr	  s    !!###r"   raise_windowc                   	 |                                  }n## t          $ r |                                   w xY w|                     ||          }|r|d         nd}|rp|d         | _        |d         | _        i | _        |d         p|| _        | j        | j        d| _        t          ddd	|d          d
| j         d| j         d          S |                                  t          ddd| d          S )u`  Target an app for subsequent actions without stealing system focus.

        cua-driver background-automation never needs to bring a window to the
        front: capture(app=...) already selects the right window via
        list_windows. We implement focus_app as a pure window-selector —
        enumerate on-screen windows, find the best match for *app*, and store
        its pid/window_id so that subsequent click/type calls hit the right
        process.

        raise_window=True is intentionally ignored: stealing the user's focus
        is exactly what this backend is designed to avoid.
        r   NrL  r   rM  r  T	focus_appz	Targeted z (pid z	, window z) without raising window.r  Fz#No on-screen window found for app 'r  )
r  rE   rq  r  rX  rY  r\  rZ  r[  r	   )rj  rt  r	  rl   matchedr   s         r    r  zCuaDriverBackend.focus_appu	  sk   	((**GG 	 	 	%%'''	 --gs;;
  '0D 	%e}D%+K%8D"$&D!#J/63DN'!3! !D  TF:$6 T Td>N T T"&"8T T T   
 	!!###u[$Q#$Q$Q$QS S S 	Ss     7)r  r   urlsadditional_arguments creates_new_application_instancer  r  r  r  c                  |s|st          d          d| j        i}|r||d<   |r||d<   |rt          |          |d<   |rt          |          |d<   |rd|d<   | j                            d	|          }|d
         p	d|d         iS )u  Idempotent launch. Returns ``{pid, bundle_id, name, windows[]}``
        so callers can skip an extra ``list_windows`` round-trip before
        ``get_window_state``.

        ``creates_new_application_instance=True`` forces a new instance
        even if the app is already running — use it when concurrent
        runs may touch the same app so each session gets its own
        isolated window.z,launch_app requires either bundle_id or namer  r  r   r  r  Tr  
launch_appr
  r   )ra   r`  r   r  r  )rj  r  r   r  r  r  r   r   s           r    r  zCuaDriverBackend.launch_app	  s    "  	M 	MKLLL )4+;< 	* )D 	 DL 	&::DL 	F+/0D+E+ED'(+ 	<7;D34m%%lD99&'@FCK+@@r"   c               L    |                      ddt          |          i          S )zYTerminate by pid. Equivalent to ``kill -9`` on POSIX,
        ``taskkill /F`` on Windows.kill_apprL  r  r_   )rj  rL  s     r    r  zCuaDriverBackend.kill_app	  s#     ||JC(9:::r"   )r   c               x    dt          |          i}|t          |          |d<   |                     d|          S )zActivate a window so subsequent foreground-dispatched input
        lands on it. cua-driver's docstring notes this is the cheaper
        path than per-call SetForegroundWindow flashes.rL  Nr   r  )r_   r  )rj  rL  r   r   s       r    r  zCuaDriverBackend.bring_to_front	  s?    
 !&s3xx0  #ID||,d333r"   c                h    |                      dt          |          t          |          d          S )uH  Move the agent-cursor *overlay* to a screen point. This is a
        visual hint — it does NOT move the real OS pointer (cua-driver
        explicitly avoids stealing pointer focus). The overlay glides
        smoothly to the target, so consumers use it before a click to
        give a visible "where the agent is going" cue.move_cursor)r  r  r  )rj  r  r  s      r    r  zCuaDriverBackend.move_cursor	  s,     ||MQc!ff+E+EFFFr"   r&  c                    | j                             dd| j        i          }|                    d          pi }t	          |                    dd                    t	          |                    dd                    fS )zPReturn the *real* OS cursor position in screen points
        (origin top-left).get_cursor_positionr  r
  r  r   r  )r  r  r`  r   r_   )rj  r   r   s      r    r  z$CuaDriverBackend.get_cursor_position	  st     m%%!It/?#@
 
 WW())/R266#q>>""CsA$7$777r"   c                t    | j                             dd| j        i          }|                    d          pi S )zReturn the logical size of the main display in points plus
        its backing scale factor. Shape:
        ``{width, height, backing_scale_factor}``.get_screen_sizer  r
  r  r  r`  r   rj  r   s     r    r  z CuaDriverBackend.get_screen_size	  sB     m%%	4+;<
 
 ww*++1r1r"   g      ?r  r  )factorr  r  r   rs   r  r  r  r  c                  | j                             dt          |          t          |          t          |          t          |          t          |          t          |          |t          |          | j        d	          S )zReturn a JPEG / PNG of a sub-region of a window, optionally
        scaled. cua-driver supports zoom-to-rect for callers that need
        a higher-resolution view of a specific element.zoom)	r   r  r  rs   r  r  r  r  r  )r  r  r_   r   r`  )	rj  r   r  r  rs   r  r  r  r  s	            r    r   zCuaDriverBackend.zoom	  sn     }&&vYqauQxxeAhhFmmW'0
 0
   	r"   rd  enabledre  c               ^    dt          |          i}|r||d<   |                     d|          S )z:Toggle the agent cursor overlay's visibility for this run.r!  re  rj  )r   r  )rj  r!  re  r   s       r    rj  z)CuaDriverBackend.set_agent_cursor_enabled
  s;     !*4==9 	* )D||6===r"   )glide_msdwell_msidle_hide_msre  r#  rz  r$  r%  c                   i }|t          |          |d<   |t          |          |d<   |t          |          |d<   |r||d<   |                     d|          S )u   Tune the overlay's motion timings — glide duration, post-click
        dwell, idle-hide delay. Each None means "leave at current value".Nr#  r$  r%  re  set_agent_cursor_motion)r   r  )rj  r#  r$  r%  re  r   s         r    r'  z(CuaDriverBackend.set_agent_cursor_motion
  st      "$XD$XD##(#6#6D  	* )D||5t<<<r"   )gradient_colorsbloom_color
image_pathre  r(  r)  r*  c                   i }|t          |          |d<   |||d<   |||d<   |r||d<   |                     d|          S )u   Customise the cursor body. ``gradient_colors`` are CSS hex
        strings tip→tail; ``bloom_color`` is the radial halo; an
        ``image_path`` (.svg/.png/.ico) replaces the silhouette
        entirely. Empty values revert to the palette default.Nr(  r)  r*  re  set_agent_cursor_style)r   r  )rj  r(  r)  r*  re  r   s         r    r,  z'CuaDriverBackend.set_agent_cursor_style"
  si      "&&*?&;&;D"#""-D!!+D 	* )D||4d;;;r"   c                   d| j         i}|r||d<   | j                            d|          }|                    d          pi S )zReturn ``{x, y, config: {cursor_color, cursor_icon, ...},
        enabled}`` for this run's cursor (or the named ``cursor_id``).r  re  get_agent_cursor_stater
  )r`  r  r  r   )rj  re  r   r   s       r    r.  z'CuaDriverBackend.get_agent_cursor_state6
  sS     !*4+;< 	* )Dm%%&>EEww*++1r1r"   )record_video
output_dirr/  c                   | j                             d|t          |          | j        d          }|                    d          pi S )a;  Enable trajectory recording (per-turn screenshots + action
        JSON) to ``output_dir``. ``record_video=True`` ALSO captures
        the main display to ``<output_dir>/recording.mp4`` (H.264).
        Recording ownership is keyed by this run's session id so
        concurrent runs don't fight over the recorder.start_recording)r0  r/  r  r
  )r  r  r   r`  r   )rj  r0  r/  r   s       r    r2  z CuaDriverBackend.start_recordingB
  sV     m%%&7$ ..':
 :
  
 ww*++1r1r"   c                t    | j                             dd| j        i          }|                    d          pi S )zDisable recording and finalise the mp4 (if video was on).
        Returns the recorder's final state including ``last_video_path``.stop_recordingr  r
  r  r  s     r    r4  zCuaDriverBackend.stop_recordingP
  sE     m%%&6t'9
   ww*++1r1r"   c                t    | j                             dd| j        i          }|                    d          pi S )zReturn the current recorder state without changing it.
        Shape: ``{recording, enabled, output_dir, next_turn,
        last_video_path, last_error, owner, video_active}``.get_recording_stater  r
  r  r  s     r    r6  z$CuaDriverBackend.get_recording_stateX
  sB     m%%!It/?#@
 
 ww*++1r1r"   )dry_runspeed_factortrajectory_dirr7  r8  c                   | j                             d|t          |          t          |          | j        d          S )zReplay a prior recording's turn stream by re-invoking each
        turn's tool call in lexical order. ``dry_run=True`` logs without
        actually firing the tools.replay_trajectory)r9  r7  r8  r  )r  r  r   r   r`  )rj  r9  r7  r8  s       r    r;  z"CuaDriverBackend.replay_trajectorya
  sH     }&&':,G}}!,//'	=
 =
   	r"   c                F    | j                             dd| j        i          S )zBootstrap ffmpeg for ``start_recording(record_video=True)``
        on Linux / Windows. macOS records natively via ScreenCaptureKit
        and doesn't need ffmpeg.install_ffmpegr  )r  r  r`  ri  s    r    r=  zCuaDriverBackend.install_ffmpegn
  s+     }&&y$*:;
 
 	
r"   c                t    | j                             dd| j        i          }|                    d          pi S )z-Return the current cua-driver runtime config.
get_configr  r
  r  r  s     r    r?  zCuaDriverBackend.get_configx
  s@    m%%9d&67
 
 ww*++1r1r"   c                H    |                      dt          |                    S )u   Set cua-driver config keys. Common keys include
        ``max_image_dimension`` (image-output resizing), recording
        flags, etc. Unknown keys are passed through verbatim — cua-driver
        validates against its own schema.ri  )r  r-   )rj  configs     r    ri  zCuaDriverBackend.set_config
  s    
 ||L$v,,777r"   c                    | j                             dd| j        i          }|                    d          p	d|d         iS )a.  Return a lightweight snapshot of running regular apps +
        on-screen visible windows with bounds, z-order, owner pid.
        Roughly the data ``list_windows`` exposes, in one call. Most
        callers should prefer ``capture()`` / ``focus_app()`` which
        already use this shape internally.get_accessibility_treer  r
  r   r  r  s     r    rC  z'CuaDriverBackend.get_accessibility_tree
  sK     m%%$y$2B&C
 
 ww*++DF/DDr"   	page_argsr   c                   t          |          || j        d}|                    |           | j                            d|          S )az  Interact with a browser page loaded in a running app (Chrome,
        Safari, Edge, ...). cua-driver routes through CDP / Apple Events
        / AX tree depending on the target. ``action`` + ``page_args``
        shape depends on the requested operation (e.g. ``action="eval"``
        takes ``js: str``); see cua-driver's ``page`` tool description
        for the full grammar.)rL  r*   r  page)r_   r`  updater  r  )rj  rL  r*   rD  r   s        r    rF  zCuaDriverBackend.page
  sN     s88' 
  

 	I}&&vt444r"   ry  r   r   r   c                   |rt          |          ni }|                    d| j                   | j                            |||          S )u  Call any cua-driver MCP tool by name with arbitrary args.
        ``session`` is injected (preserves the caller's explicit one
        via setdefault). For tools the wrapper doesn't already type-
        wrap, this is the supported escape hatch — preferred over
        reaching for ``self._session.call_tool`` directly because it
        keeps the session-id contract consistent with everything else.r  r   )r-   
setdefaultr`  r  r  )rj  r   r   r   payloads        r    r  zCuaDriverBackend.call_tool
  sN     !%,$t***"9d&6777}&&tWg&FFFr"   r  c                    |                     d          }t          |t                    sdS | j                             |          }|sdS | j                            d|          sdS ||d<   dS )a  Surface 6: when the wrapper is about to call a token-capable
        tool with `element_index`, look up the matching `element_token`
        from the last snapshot and attach it. cua-driver-rs's contract
        for combined args is documented in trycua/cua#1961:

          "element_token takes precedence over element_index when both
           supplied. Returns an explicit 'stale' error if the snapshot
           has been superseded."

        Gated on the per-tool capability claim so we don't send the
        field to drivers that predate the surface (which would reject
        the schema with `additionalProperties: false`).
        r  Nzaccessibility.element_tokensr  r  )r   r,   r_   r\  r  r  )rj  r  r   r!  r#  s        r    _maybe_attach_element_tokenz,CuaDriverBackend._maybe_attach_element_token
  s     hh''#s## 	F%))#.. 	F}00* 1 
 
 	 F %_r"   c           
        |                      ||           |                    d| j                   	 | j                            ||          }nG# t
          $ r:}t                              d|           t          d|d|           cY d }~S d }~ww xY w|d          }|d         }|	                    d          pi }d	}t          |t                    r$t          |	                    d
d	                    }nt          |t                    r|}|s8t          |t                    r#t          |	                    d
d	                    }i }	t          |t                    r|	                    |           t          |t                    r|	                    |           t          ||||	||	                    d                    S )Nr  zcua-driver %s call failedFzcua-driver error: r  r  r   r
  rz   r   r+   r   )rL  rI  r`  r  r  rE   r   	exceptionr	   r   r,   r-   r   rG  r.   )
rj  r   r   r   r  r   r   r   r   r   s
             r    r  zCuaDriverBackend._action
  s    	((t444
 		4#3444	Y-))$55CC 	Y 	Y 	Y8$???5?WTU?W?WXXXXXXXXX	Y Y6{WW0117R
dD!! 	$((9b1122GGc"" 	G 	9:j$77 	9*..B7788G  "dD!! 	KKj$'' 	$KK
###"4WdJ6:hh6O6OQ Q Q 	Qs   A 
B/BBBr   r.  )rz   )rr  r   r   r   r   r
   r-  )r   rm   )rl   rm   rt  r   r   rm   )r  NNN)
rr  r   rt  r   rL  r\   r   r\   r   r
   )
r*   r   r   r   r+   r   r  r   r   r  )r  r\   r  r\   r  r\   r  r   r  r_   r^  r  r+   r   r  r   r   r	   )r  r\   r  r\   r  r  r  r  r  r   r^  r  r+   r   r  r   r   r	   )r  r   r  r_   r  r\   r  r\   r  r\   r^  r  r+   r   r  r   r   r	   )r   r   r+   r   r  r   r   r	   )rP  r   r+   r   r  r   r   r	   r   )r   r   r  r\   r   r	   )F)rt  r   r	  r   r   r	   )r  r   r   r   r  r  r  r  r  r   r   r   )rL  r_   r   r	   )rL  r_   r   r\   r   r	   )r  r_   r  r_   r   r	   )r   r&  r   r   )r   r_   r  r   r  r   rs   r   r  r   r  r   r  r   r  r_   r   r   )r!  r   re  r   r   r	   )
r#  rz  r$  rz  r%  rz  re  r   r   r	   )
r(  r  r)  r   r*  r   re  r   r   r	   )re  r   r   r   )r0  r   r/  r   r   r   )r9  r   r7  r   r8  r   r   r   )r   r	   )rL  r_   r*   r   rD  r   r   r   )r   r   r   r   r   r   r   r   )r  r   r   r   r   r   )r   r   r   r   r   r	   )/r  r  r  r  rk  r  r  ro  rq  rw  ry  r  r  r  r  r  r  r  r  r   r  r  r{  r  r  r  r  r  r  r  r   rj  r'  r,  r.  r2  r4  r6  r;  r=  r?  ri  rC  rF  r  rL  r  rZ   r"   r    rU  rU    s(       JJ#B #B #B #BL3V 3V 3V 3Vj        - - - -# # # #
 
 
 
 
    ( ( ( (TM
 M
 M
 M
d !!#'P
 P
 P
 P
 P
f
( ( ( (Z "&)-'+$6( 6( 6( 6( 6( 6(v '+$(-1+/)-'+$%* %* %* %* %* %*V !%)-'+$+, +, +, +, +, +,\ FJ)./ / / / / / @D#(3 3 3 3 3 3:/ / / / /&   ,$ $ $ $)S )S )S )S )Sh $("$(4816A A A A A AB; ; ; ; 374 4 4 4 4 4G G G G8 8 8 82 2 2 2 !     . =A> > > > > > =A<@@D;?	= = = = = =& GK<@;?:>	< < < < < <* ;?2 2 2 2 2 2 .32 2 2 2 2 22 2 2 22 2 2 2 +003     
 
 
 
2 2 2 28 8 8 8	E 	E 	E 	E5 5 5 5$
G&*
G 
G 
G 
G 
G 
G& & & &4 Q  Q  Q  Q  Q  Qr"   rU  )r   r   r   r   r   r   r   r   r   r   r   r   r   r	   rO  r.  )r   r\   r   )rd   re   r   rf   )rl   rm   r   r   )rx   r   r   r\   r  )rl   rm   r   r   r   r   r   r   )r   r   r   r   r   r   )r   r   r   r   r   r   )r   r   r   r   )r   r   r   r   )r   r   r   r   )r   r   r   r   )r   r   r   r   )r   r   r   r/  )r  r   r   r  )r  rm   r   r  )r   r%  r   r&  )rG  r   r   rH  )rP  r   r   rQ  )r3  r   r   r   )r   r   r   rA  )r   r   r   r\   )rJ  rm   r   rm   )Nr  
__future__r   rn  r  concurrent.futuresr  	functoolsr   loggingrP   r|   r   r   rN   r   r]  typingr   r   r   r   r   tools.computer_use.backendr	   r
   r   r   	getLoggerr  r   r.   r   r   r   r  r  r   ri   rF   rW   r[   rc   rk   rw   r   r   r   r   r   r   	lru_cacher   compile	MULTILINEr  r   r   r   r   r   r   r   r   r  r  r  r$  rF  rO  ra  rc  r  r  rG  rI  rS  rU  rZ   r"   r    <module>rZ     sA  ! !F # " " " " "             				 				      



      3 3 3 3 3 3 3 3 3 3 3 3 3 3            
	8	$	$ )-< < < < < <\ . & 7  VUU  "  ;       <E E E E
$ 
$ 
$ 
$    < < < <   $= = = =&    	           L CJ CJ CJ CJ CJ CJP .     Q    T 2:. L	 	  $ $ $ $M M M M! ! ! ! !H    &0 0 0 0
 14 ) ) ) ) ) )X       *   *   82 2 2 2j$ $ $ $N      2/ / / / / / / /l}X }X }X }X }X }X }X }X@5 5 5 5p$ $ $ $N* * * *' ' ' '\pQ pQ pQ pQ pQ) pQ pQ pQ pQ pQr"   