
    ߩ^j$                        d Z ddlmZ ddlmZmZ ddlmZmZ ddl	m
Z
mZmZmZmZ e G d d                      Ze G d d	                      Ze G d
 d                      Z G d de          ZdS )zAbstract backend interface for computer use.

Any implementation (cua-driver over MCP, pyautogui, noop, future Linux/Windows)
must return the shape described below. All methods synchronous; async is
handled inside the backend implementation if needed.
    )annotations)ABCabstractmethod)	dataclassfield)AnyDictListOptionalTuplec                      e Zd ZU dZded<   ded<   dZded<   dZd	ed
<   dZded<   dZded<   dZ	ded<    e
e          Zded<   dZded<   ddZdS )	UIElementz/One interactable element on the current screen.intindexstrrole label)r   r   r   r   zTuple[int, int, int, int]boundsappr   pid	window_iddefault_factoryDict[str, Any]
attributesNOptional[str]element_tokenreturnTuple[int, int]c                :    | j         \  }}}}||dz  z   ||dz  z   fS )N   )r   )selfxywhs        </home/ice/.hermes/hermes-agent/tools/computer_use/backend.pycenterzUIElement.center#   s,    [
1a16z1qAv:%%    )r   r    )__name__
__module____qualname____doc____annotations__r   r   r   r   r   r   dictr   r   r)    r*   r(   r   r      s         99JJJIIIEOOOO(4F4444CMMMMCLLLLI!&t!<!<!<J<<<< $(M''''& & & & & &r*   r   c                      e Zd ZU dZded<   ded<   ded<   dZded	<    ee
          Zded<   dZ	ded<   dZ
ded<   dZded<   dZded<   dS )CaptureResultu  Result of a screen capture call.

    At least one of png_b64 / elements is populated depending on capture mode:
      * mode="vision" → png_b64 only
      * mode="ax"     → elements only
      * mode="som"    → both (default): PNG already has numbered overlays
                         drawn by the backend, and `elements` holds the
                         matching index → element mapping.
    r   moder   widthheightNr   png_b64r   zList[UIElement]elementsr   r   window_titler   png_bytes_lenimage_mime_type)r+   r,   r-   r.   r/   r7   r   listr8   r   r9   r:   r;   r1   r*   r(   r3   r3   (   s           IIIJJJKKK!G!!!! %d ; ; ;H;;;;CMMMMLM &*O))))))r*   r3   c                      e Zd ZU dZded<   ded<   dZded<   dZd	ed
<    ee          Z	ded<   dZ
ded<   dZded<   dZded<   dZded<   dZded<   dZded<   dZded<   dS )ActionResultu=  Result of any action (click / type / scroll / drag / key / wait).

    Beyond the transport-level ``ok`` flag, this carries cua-driver's
    structured action verdict so the model can follow the documented
    verify → escalate ladder (NousResearch/hermes-agent#67052). ``ok`` stays
    tool/transport success only — it is NOT the semantic verdict. Read
    ``effect`` / ``escalation`` to decide the next rung. All structured
    fields are optional and additive: an older driver that omits
    ``structuredContent`` leaves them ``None`` and behavior is unchanged.
    boolokr   actionr   messageNzOptional[CaptureResult]capturer   r   metazOptional[bool]verifiedr   effectzOptional[Dict[str, Any]]
escalationpathdegradeddelivery_modecode)r+   r,   r-   r.   r/   rB   rC   r   r0   rD   rE   rF   rG   rH   rI   rJ   rK   r1   r*   r(   r>   r>   F   s         	 	 HHHKKKG (,G++++ 5666D6666  $H#### F     ,0J////D#H#####'M'''' Dr*   r>   c            
         e Zd ZdZed>d            Zed>d            Zed?d            Ze	 	 	 	 d@dAd            Zed	d	d	ddd	d	dddBd"            Z	ed	d	d	d	dd	d	dd#dCd)            Z
ed*d	d	d	d	d	dd+dDd.            Zed	dd/dEd1            Zed	dd/dFd3            ZedGd5            ZdGd6ZedHdId8            ZedJdKd:            ZdLd=Zd	S )MComputerUseBackendz<Lifecycle: `start()` before first use, `stop()` at shutdown.r   Nonec                    d S Nr1   r#   s    r(   startzComputerUseBackend.startr   s     Sr*   c                    d S rP   r1   rQ   s    r(   stopzComputerUseBackend.stopu   s    Cr*   r?   c                    dS )zReturn True if the backend can be used on this host right now.

        Used by check_fn gating and by the post-setup wizard.
        Nr1   rQ   s    r(   is_availablezComputerUseBackend.is_availablex         r*   somNr4   r   r   r   r   Optional[int]r   r3   c                    d S rP   r1   )r#   r4   r   r   r   s        r(   rC   zComputerUseBackend.capture   s	     r*   left   F)elementr$   r%   buttonclick_count	modifiersrJ   bring_to_frontr]   r$   r%   r^   r_   r   r`   Optional[List[str]]rJ   ra   r>   c                   d S rP   r1   )	r#   r]   r$   r%   r^   r_   r`   rJ   ra   s	            r(   clickzComputerUseBackend.click   	     sr*   )from_element
to_elementfrom_xyto_xyr^   r`   rJ   ra   rf   rg   rh   Optional[Tuple[int, int]]ri   c                   d S rP   r1   )	r#   rf   rg   rh   ri   r^   r`   rJ   ra   s	            r(   dragzComputerUseBackend.drag   re   r*      )amountr]   r$   r%   r`   rJ   ra   	directionrn   c                   d S rP   r1   )	r#   ro   rn   r]   r$   r%   r`   rJ   ra   s	            r(   scrollzComputerUseBackend.scroll   re   r*   )rJ   ra   textc                   d S rP   r1   )r#   rr   rJ   ra   s       r(   	type_textzComputerUseBackend.type_text   s    ADr*   keysc                   dS )z7Send a key combo, e.g. 'cmd+s', 'ctrl+alt+t', 'return'.Nr1   )r#   ru   rJ   ra   s       r(   keyzComputerUseBackend.key   rW   r*   List[Dict[str, Any]]c                    dS )z9Return running apps with bundle IDs, PIDs, window counts.Nr1   rQ   s    r(   	list_appszComputerUseBackend.list_apps   rW   r*   c                    g S )zReturn visible native windows with PID and window identifiers.

        Optional compatibility hook: backends that predate window discovery
        remain instantiable and simply report no windows.
        r1   rQ   s    r(   list_windowszComputerUseBackend.list_windows   s	     	r*   raise_windowc                    dS )zJRoute input to `app` (by name or bundle ID). Default: focus without raise.Nr1   )r#   r   r}   s      r(   	focus_appzComputerUseBackend.focus_app   rW   r*   valuec                    dS )zSet a native value on an element (e.g. AXPopUpButton selection).

        `element` is the 1-based SOM index returned by a prior capture call.
        Nr1   )r#   r   r]   s      r(   	set_valuezComputerUseBackend.set_value   rW   r*   secondsfloatc           	         ddl }|                    t          dt          |d                               t	          ddd|dd	
          S )z#Default implementation: time.sleep.r   Ng        g      >@Twaitzwaited z.2fs)r@   rA   rB   )timesleepmaxminr>   )r#   r   r   s      r(   r   zComputerUseBackend.wait   sU    

3sC..//000tF<Tg<T<T<T<TUUUUr*   )r   rN   )r   r?   )rX   NNN)
r4   r   r   r   r   rY   r   rY   r   r3   )r]   rY   r$   rY   r%   rY   r^   r   r_   r   r`   rb   rJ   r   ra   r?   r   r>   )rf   rY   rg   rY   rh   rj   ri   rj   r^   r   r`   rb   rJ   r   ra   r?   r   r>   )ro   r   rn   r   r]   rY   r$   rY   r%   rY   r`   rb   rJ   r   ra   r?   r   r>   )rr   r   rJ   r   ra   r?   r   r>   )ru   r   rJ   r   ra   r?   r   r>   )r   rx   )F)r   r   r}   r?   r   r>   rP   )r   r   r]   rY   r   r>   )r   r   r   r>   )r+   r,   r-   r.   r   rR   rT   rV   rC   rd   rl   rq   rt   rw   rz   r|   r   r   r   r1   r*   r(   rM   rM   o   sm       FF    ^  ^   ^  !!#'    ^  "&)-'+$     ^  '+$(-1+/)-'+$     ^ 
 !%)-'+$     ^ EI).E E E E E ^E ?C#(F F F F F ^F
 H H H ^H    Y Y Y Y ^Y     ^V V V V V Vr*   rM   N)r.   
__future__r   abcr   r   dataclassesr   r   typingr   r	   r
   r   r   r   r3   r>   rM   r1   r*   r(   <module>r      sb    # " " " " " # # # # # # # # ( ( ( ( ( ( ( ( 3 3 3 3 3 3 3 3 3 3 3 3 3 3 & & & & & & & &0 * * * * * * * *: % % % % % % % %PmV mV mV mV mV mV mV mV mV mVr*   