
    .cj{                    x   U d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	m
Z
 ddlmZmZmZmZmZ ddlmZ ddlmZ ddlmZ  ej        e          ZeZdad	ed
<    ej                    Zdaded<    ej                    Z i Z!ded<   dZ"dZ#dZ$dZ%dZ& ej                    Z'd Z(d_dZ)d`dZ*dadZ+dbdZ,dcd"Z-dad#Z.ddd$Z/ddd%Z0ded'Z1dfd)Z2dgd-Z3dfd.Z4dfd/Z5dfd0Z6dhd1Z7dhd2Z8did4Z9djd7Z:ddd8Z;dkd9Z<dbd:Z=dkd;Z>dd<d<de"d=dldMZ?dmdOZ@dndPZAdd<d<de"ddQdodTZBdpdVZCdqdXZDdrdsd[ZE	 	 	 	 dtdud]ZFdbd^ZGdS )vu  
Async (background) delegation registry.

Backs ``delegate_task(background=true)``: the parent agent dispatches a
subagent that runs on a module-level daemon executor and returns a handle
immediately, so the user and the model can keep working while the child runs.

When the child finishes, a completion event is pushed onto the SHARED
``process_registry.completion_queue`` with ``type="async_delegation"``. The
CLI (``cli.py`` process_loop) and gateway (``_run_process_watcher`` /
``completion_queue`` drain) already poll that queue while the agent is idle
and forge a fresh user/internal turn from each event. We deliberately reuse
that rail rather than reaching into a running agent loop:

  - completions surface as a NEW turn when the agent is idle, never spliced
    between a tool result and an assistant message. That keeps strict
    message-role alternation legal and the prompt cache intact (hard
    invariant: never mutate past context).
  - we inherit the queue's de-dup, crash-recovery checkpoint, and the
    existing CLI + gateway drain wiring for free — no new drain loops in the
    two largest files in the repo.

The completion payload carries a RICH, self-contained task-source block (the
original goal, the context the parent supplied, toolsets, model, dispatch
time, status, and the full result summary). When the result re-enters the
conversation the parent may be deep in unrelated context and won't remember
why the subagent existed; the block lets it either use the result or
re-dispatch if the world has moved on.

This module owns ONLY the async lifecycle. The actual child build + run is
delegated back to ``delegate_tool._run_single_child`` via an injected
runner, so all the credential leasing, heartbeat, timeout, and result-shaping
logic stays in one place.
    )annotationsN)ThreadPoolExecutor)AnyCallableDictListOptionalget_hermes_home)DaemonThreadPoolExecutor)propagate_context_to_threadzOptional[ThreadPoolExecutor]	_executorint_executor_max_workerszDict[str, Dict[str, Any]]_records   2   i:	 i     c                 $    t                      dz  S )Nzstate.dbr
        8/home/ice/.hermes/hermes-agent/tools/async_delegation.py_db_pathr   X   s    z))r   returnsqlite3.Connectionc                 b   ddl m}  t                      }|j                            dd           t          j        |d          } | |d           |                    d	           d
 |                    d          D             }dD ]$\  }}||vr|                    d| d|            %|S )Nr   )apply_wal_with_fallbackT)parentsexist_ok
   )timeoutzstate.db (async_delegation))db_labela+  CREATE TABLE IF NOT EXISTS async_delegations (
            delegation_id TEXT PRIMARY KEY,
            origin_session TEXT NOT NULL,
            origin_ui_session_id TEXT NOT NULL DEFAULT '',
            parent_session_id TEXT,
            state TEXT NOT NULL,
            dispatched_at REAL NOT NULL,
            completed_at REAL,
            updated_at REAL NOT NULL,
            event_json TEXT,
            result_json TEXT,
            delivery_state TEXT NOT NULL DEFAULT 'pending',
            delivery_attempts INTEGER NOT NULL DEFAULT 0,
            delivered_at REAL,
            owner_pid INTEGER,
            owner_started_at INTEGER,
            task_json TEXT,
            delivery_claim TEXT,
            delivery_claimed_at REAL,
            origin_session_id TEXT NOT NULL DEFAULT ''
        )c                    h | ]
}|d          S )   r   ).0rows     r   	<setcomp>z_connect.<locals>.<setcomp>z   s    VVV#s1vVVVr   z$PRAGMA table_info(async_delegations)))	owner_pidINTEGER)owner_started_atr)   )	task_jsonTEXT)delivery_claimr,   )delivery_claimed_atREAL)origin_session_idr,   z)ALTER TABLE async_delegations ADD COLUMN  )hermes_stater   r   parentmkdirsqlite3connectexecute)r   pathconncolumnsnamesql_types         r   _connectr=   \   s    444444::DKdT222?4,,,DD+HIIIILL	  . WV.T!U!UVVVG X Xh wLLVTVVHVVWWWKr   recordDict[str, Any]Nonec                    t          j                     }	 ddlm}  |t          d                                                    }n# t
          $ r d }Y nw xY w fddD             }t          5  t                      5 }|                    d d          	                    dd	           	                    d
d	           	                    d           d         |t          d                                          |t          j        |           	                    dd	          f
           d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t                       d S )Nr   )get_process_start_timeosc                D    i | ]}|v |                     |          S r   get)r%   keyr>   s     r   
<dictcomp>z%_persist_dispatch.<locals>.<dictcomp>   s6       &== 	VZZ__==r   )goalgoalscontexttoolsetsrolemodelis_batcha~  INSERT OR REPLACE INTO async_delegations
               (delegation_id, origin_session, origin_ui_session_id,
                parent_session_id, state, dispatched_at, updated_at,
                delivery_state, delivery_attempts, owner_pid,
                owner_started_at, task_json, origin_session_id)
               VALUES (?, ?, ?, ?, 'running', ?, ?, 'pending', 0, ?, ?, ?, ?)delegation_idsession_key origin_ui_session_idparent_session_iddispatched_atr0   )timegateway.statusrB   
__import__getpid	Exception_DB_LOCKr=   r7   rF   jsondumps_prune_durable_records)r>   nowrB   r*   task_payloadr9   s   `     r   _persist_dispatchra      s   
)++C 99999911*T2B2B2I2I2K2KLL          X  L
 
 
 
8:: 
Q O$fjj&C&CZZ.33VZZ@S5T5TO$c:d+;+;+B+B+D+Dtz,77ZZ+R00	2	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 sH   0A AA.E=B-D6*E6D:	:E=D:	>EEErP   strc                    t           5  t                      5 }|                    d| f           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )Nz3DELETE FROM async_delegations WHERE delegation_id=?)r[   r=   r7   rP   r9   s     r   _delete_durable_delegationre      s   	 ^ ^8:: ^J]L\]]]^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^s.   A;A?	A?	AAAc                 r   t          j                     } | t          z
  }t          5  t                      5 }|                    d|f           |                    d                                          d         }t          d|t          z
            }|r|                    d|f           |                    d                                          d         }t          d|t          z
            }|r|                    d|f           ddd           n# 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )zBBound terminal history, preferring delivered records for deletion.zQDELETE FROM async_delegations WHERE delivery_state='delivered' AND updated_at < ?zRSELECT COUNT(*) FROM async_delegations WHERE state NOT IN ('running','finalizing')r   aX  DELETE FROM async_delegations WHERE delegation_id IN (
                     SELECT delegation_id FROM async_delegations
                     WHERE state NOT IN ('running','finalizing')
                     ORDER BY CASE delivery_state WHEN 'delivered' THEN 0 ELSE 1 END,
                              updated_at ASC LIMIT ?
                   )z~SELECT COUNT(*) FROM async_delegations
               WHERE state NOT IN ('running','finalizing') AND delivery_state='pending'a  DELETE FROM async_delegations WHERE delegation_id IN (
                     SELECT delegation_id FROM async_delegations
                     WHERE state NOT IN ('running','finalizing') AND delivery_state='pending'
                     ORDER BY updated_at ASC LIMIT ?
                   )N)	rV   _DURABLE_RETENTION_SECONDSr[   r=   r7   fetchonemax_MAX_RETAINED_COMPLETED_MAX_DURABLE_PENDING)r_   cutoffr9   terminal_countexcesspending_countoverflows          r   r^   r^      s   
)++C--F	    8::  _I	
 	
 	
 `
 

(**Q Q)@@AA 		LL 	   [
 
 (**Q q-*>>?? 	LL
   3                                                                 s5   D,CDD,D	D,D	D,,D03D0eventresultc                   t          j                     }t          5  t                      5 }|                    d|                     dd          |                     d|          |t          j        |           t          j        |          | d         f           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )NzUPDATE async_delegations SET state=?, completed_at=?, updated_at=?,
               event_json=?, result_json=?, delivery_state='pending'
               WHERE delegation_id=?status	completedcompleted_atrP   )rV   r[   r=   r7   rF   r\   r]   )rq   rr   r_   r9   s       r   _persist_completionrw      s8   
)++C	 
 
8:: 
( YYx--uyy/M/MsZ
6 2 2E/4JL		
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s5   B=A/B%B=%B)	)B=,B)	-B==CCc                    t           5  t                      5 }|                    dt          j                    | f           d d d            n# 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )NzfUPDATE async_delegations SET delivery_attempts=delivery_attempts+1, updated_at=? WHERE delegation_id=?)r[   r=   r7   rV   rd   s     r   _note_delivery_attemptry      s    	 
 
8:: 
tY[[-(	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s4   A%*AA%A	A%A	A%%A),A)c                 f   	 ddl m} m} n# t          $ r Y dS w xY wt	          j                    }d}t
          5  t                      5 }|                    d                                          }|D ]}|\	  }}}	}
}}}}}d}|rE | t          |                    }|r+|) |t          |                    t          |          k    }|r[t          j        |pd          }i ddd	|d
|d|	d|pdd|
d|                    dd          d|                    d          d|                    d          d|                    d          d|                    d          d|                    d          dt          |                    d                    ddddddd|d|i}dd|d         d}|                    d||t          j        |          t          j        |          |f           |dz  }	 ddd           n# 1 swxY w Y   ddd           n# 1 swxY w Y   |S ) zEClassify records whose owning process disappeared as outcome unknown.r   )_pid_existsrB   a  SELECT delegation_id, origin_session, origin_ui_session_id,
                      parent_session_id, dispatched_at, owner_pid,
                      owner_started_at, task_json, origin_session_id
               FROM async_delegations WHERE state IN ('running','finalizing')FNz{}typeasync_delegationrP   rQ   rS   r0   rR   rT   rI   rJ   rK   rL   rM   rN   rO   rt   unknownsummaryerrorzLDelegation owner exited before recording a terminal result; outcome unknown.rU   rv   )rt   r   r   zUPDATE async_delegations SET state='unknown', completed_at=?,
                   updated_at=?, event_json=?, result_json=?, delivery_state='pending'
                   WHERE delegation_id=?r$   )rW   r{   rB   rZ   rV   r[   r=   r7   fetchallr   r\   loadsrF   boolr]   )r{   rB   r_   	recoveredr9   rowsr&   rP   rQ   	origin_ui	parent_idrU   pidstartedr+   r0   livetaskrq   rr   s                       r   recover_abandoned_delegationsr      s   FFFFFFFFF   qq
)++CI	 ' '8:: '||Q
 

 (** 	   	  	C;>9]KI}'9&7D L"{3s88,, LG/11#c((;;s7||KD :i/400D*,;]{,BI
 $%6%<" $Y 178L8L '** -6txx	7J7J DHHZ00 39$((6:J:J '** -7TXXj=Q=Q8R8R ) &/ g   1? E !*dU7^TTFLL, c4:e,,dj.@.@-P	   NIIA 	' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' 'P sB    
H&F;HH&H	H&H	H&&H*-H*c                   t                       t          5  t                      5 }|                    d                                          }|D ]H\  }}t          j        |          }t          |t                    rd|d<   | 	                    |           I	 ddd           n# 1 swxY w Y   ddd           n# 1 swxY w Y   t          |          S )uz  Enqueue durable pending completions as fresh turns after process start.

    Every restored event is stamped ``restored=True`` (in-memory only — the
    stamp is added after the durable payload is deserialized and is never
    persisted). Restored events originate from a *previous* process, so no
    consumer in THIS process implicitly owns them: drain paths that run
    without an ownership filter (the legacy single-session behavior) must
    leave them queued for a consumer that can positively prove ownership,
    otherwise a brand-new session adopts a dead session's delegation
    results seconds after boot (#64484).
    zSELECT delegation_id, event_json FROM async_delegations
               WHERE state != 'running' AND delivery_state='pending' AND event_json IS NOT NULL
               ORDER BY completed_at, delegation_idTrestoredN)r   r[   r=   r7   r   r\   r   
isinstancedictputlen)target_queuer9   r   _delegation_idpayloadevts         r   restore_undelivered_completionsr     sV    "###	 
" 
"8:: 
"||7
 
 (**	 	
 (, 	" 	"#NG*W%%C#t$$ '"&JS!!!!		"
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" 
" t99s5   B<A3B%B<%B)	)B<,B)	-B<<C C r   c                   t          j                     }t          5  t                      5 }|                    d||| f          }|j        dk    cddd           cddd           S # 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )zDAtomically acknowledge successful injection of a durable completion.zUPDATE async_delegations SET delivery_state='delivered', delivered_at=?, updated_at=?
               WHERE delegation_id=? AND delivery_state!='delivered'r$   NrV   r[   r=   r7   rowcount)rP   r_   r9   curs       r   mark_completion_deliveredr   7  s)   
)++C	 ! !8:: !llH#}%
 

 |q ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !4   A?$A'A?'A+	+A?.A+	/A??BBclaim_idc                   t          j                     }t          5  t                      5 }|                    d| f                                          }|	 ddd           ddd           dS |                    d|||| |dz
  f          }|j        dk    cddd           cddd           S # 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )zBClaim one pending completion across competing consumers/processes.zBSELECT delivery_state FROM async_delegations WHERE delegation_id=?NTa  UPDATE async_delegations SET delivery_claim=?, delivery_claimed_at=?,
                      delivery_attempts=delivery_attempts+1, updated_at=?
               WHERE delegation_id=? AND delivery_state='pending'
                 AND (delivery_claim IS NULL OR delivery_claimed_at < ?)i,  r$   )rV   r[   r=   r7   rh   r   )rP   r   r_   r9   r&   r   s         r   claim_completion_deliveryr   C  s   
)++C	 ! !8:: !llP
 
 (** 	 ;! ! ! ! ! ! ! ! ! ! ! ! ! ! llL sCc	:
 
 |q ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !s@   C-B0C/(B0C0B4	4C7B4	8CCCr   consumerOptional[str]c                .   |                      d          dk    rdS t          |                      d          pd          }|sdS | dt          d                                           dt	          j                    j         }t          ||          r|ndS )zCClaim a durable delegation event; non-durable events need no token.r|   r}   rR   rP   :rC   N)rF   rb   rX   rY   uuiduuid4hexr   )r   r   rP   r   s       r   claim_event_deliveryr   W  s    
wwv,,,r006B77M rKKZ--4466KK9IKKH0IIS88tSr   c           	        t          j                     }t          5  t                      5 }|                    d|| |t          f          }|j        dk    r:t                              d| t                     	 ddd           ddd           dS |                    d|| |f          }|j        dk    cddd           cddd           S # 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )u  Release a failed delivery claim so another consumer may retry.

    Attempts are counted at claim time, so a row that keeps being claimed and
    released has burned real delivery attempts. Once the budget is exhausted
    the row converges to a terminal ``dropped`` state instead of returning to
    ``pending`` — otherwise an undeliverable completion replays on every
    gateway restart forever (restore_undelivered_completions only restores
    pending rows).
    a	  UPDATE async_delegations SET delivery_state='dropped',
                      delivery_claim=NULL, delivery_claimed_at=NULL, updated_at=?
               WHERE delegation_id=? AND delivery_state='pending'
                 AND delivery_claim=? AND delivery_attempts>=?r$   znAsync delegation %s exhausted its %d delivery attempts; marking terminally dropped (result remains queryable).NTzUPDATE async_delegations SET delivery_claim=NULL,
                      delivery_claimed_at=NULL, updated_at=?
               WHERE delegation_id=? AND delivery_state='pending'
                 AND delivery_claim=?)rV   r[   r=   r7   _MAX_DELIVERY_ATTEMPTSr   loggerwarning)rP   r   r_   r9   cappedr   s         r   release_completion_deliveryr   b  s    )++C	 ! !8:: !B -+AB
 
 ?aNNI5  
 ! ! ! ! ! ! ! ! ! ! ! ! ! ! ll) -*
 
 |q -! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !sA   C#AC7C##C2C#C	C#C	C##C'*C'c                   t          j                     }t          5  t                      5 }|                    d|| |f          }|j        dk    cddd           cddd           S # 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )u  Terminally drop a claimed completion that can never be delivered.

    Used when the delivery target is permanently gone — the spawning session
    ended at an explicit user boundary (/new, reset) rather than a compression
    rotation. Marking the row ``dropped`` (not ``delivered``) keeps the ack
    honest, and (not ``pending``) keeps restart recovery from replaying a
    completion that will be fail-closed dropped again every time.
    a  UPDATE async_delegations SET delivery_state='dropped',
                      updated_at=?, delivery_claim=NULL,
                      delivery_claimed_at=NULL
               WHERE delegation_id=? AND delivery_state='pending'
                 AND delivery_claim=?r$   Nr   rP   r   r_   r9   r   s        r   drop_completion_deliveryr     s,    )++C	 	! 	!8:: 	!ll)
 -*
 
 |q 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	!r   c           	        t          j                     }t          5  t                      5 }|                    d||| |f          }|j        dk    cddd           cddd           S # 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )z;Acknowledge acceptance for the consumer holding this claim.a  UPDATE async_delegations SET delivery_state='delivered',
                      delivered_at=?, updated_at=?, delivery_claim=NULL,
                      delivery_claimed_at=NULL
               WHERE delegation_id=? AND delivery_state='pending'
                 AND delivery_claim=?r$   Nr   r   s        r   complete_completion_deliveryr     s,   
)++C	 	! 	!8:: 	!ll)
 #}h/
 
 |q 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	! 	!s4   B %A(B (A,	,B /A,	0B  BBc                    |rM|                      d          dk    r6t          t          |                      d          pd          |           d S d S d S Nr|   r}   rP   rR   )rF   r   rb   r   r   s     r   complete_event_deliveryr     sa     TCGGFOO'999$S)A)A)GR%H%H(SSSSST T99r   c                    |rM|                      d          dk    r6t          t          |                      d          pd          |           d S d S d S r   )rF   r   rb   r   s     r   release_event_deliveryr     sa     SCGGFOO'999#C(@(@(FB$G$GRRRRRS S99r   Optional[Dict[str, Any]]c           
        t           5  t                      5 }|                    d| f                                          }d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   |d S | |d         |d         |d         |d         |d         rt	          j        |d                   nd |d         |d         |d	         pd
d	S )NzSELECT origin_session, state, dispatched_at, completed_at,
                      result_json, delivery_state, delivery_attempts,
                      origin_session_id
               FROM async_delegations WHERE delegation_id=?r   r$      r               rR   )	rP   origin_sessionstaterU   rv   rr   delivery_statedelivery_attemptsr0   )r[   r=   r7   rh   r\   r   )rP   r9   r&   s      r   get_durable_delegationr     sV   	  8:: ll? BO@P	
 

 (** 	                              {t&#a&3q6QQ(+A8$*SV$$$Da&s1v V\r  s4   A$*AA$A	A$A	A$$A(+A(max_workersr   c                    t           5  t          | t          k    rt          | d          a| at          cddd           S # 1 swxY w Y   dS )u  Lazily create (or grow) the shared daemon executor.

    We never shrink — ThreadPoolExecutor can't resize — but if the configured
    cap grows between calls we rebuild a larger pool. Existing in-flight
    futures keep running on the old pool until it's garbage collected.
    Nzasync-delegate)r   thread_name_prefix)_executor_lockr   r   _DaemonThreadPoolExecutor)r   s    r   _get_executorr     s     
  .C C C1'#3  I %0!                 s   ,AAAc                     t           5  t          d t                                          D                       cddd           S # 1 swxY w Y   dS )z.Number of async delegations currently running.c              3  H   K   | ]}|                     d           dv dV  dS )rt   >   running
finalizingr$   NrE   r%   rs     r   	<genexpr>zactive_count.<locals>.<genexpr>  s7      ``155??F_3_3_13_3_3_3_``r   N)_records_locksumr   valuesr   r   r   active_countr     s    	 a a``hoo//`````a a a a a a a a a a a a a a a a a as   0AA	A	c                 H    dt          j                    j        d d          S )Ndeleg_r   )r   r   r   r   r   r   _new_delegation_idr     s"    *DJLL$RaR(***r   c                 .   d t                                           D             } t          |           t          k    rdS |                     d            | dt          |           t          z
           D ] \  }}t                               |d           !dS )ziDrop the oldest completed records beyond the retention cap.

    Caller must hold ``_records_lock``.
    c                N    g | ]"\  }}|                     d           dk    ||f#S rt   r   rE   )r%   ridr   s      r   
<listcomp>z+_prune_completed_locked.<locals>.<listcomp>  s?       C55??i'' 
a'''r   Nc                r    | d                              d          p| d                              d          pdS )Nr$   rv   rU   r   rE   )kvs    r   <lambda>z)_prune_completed_locked.<locals>.<lambda>  s1    "Q%))N";";"^r!uyy?Y?Y"^]^ r   )rG   )r   itemsr   rj   sortpop)ru   r   _s      r   _prune_completed_lockedr     s    
 nn&&  I
 9~~000NN^^N___Fc)nn/FFFG    QS$   r   c                 r    	 ddl m}   | dd          dk    rdS  | dd          pdS # t          $ r Y dS w xY w)u  Raw session id of the ORIGINATING api_server request, or ``""``.

    The obvious source — ``HERMES_SESSION_ID`` via ``get_session_env`` — is
    NOT safe to read at dispatch time: constructing a child agent
    (``agent/agent_init.py``) calls ``set_current_session_id(child.session_id)``,
    clobbering that ContextVar *and* ``os.environ`` with the subagent's
    internal ``{timestamp}_{uuid}`` id moments before the dispatch code reads
    it, so the completion wake would self-post into the subagent's own
    (unread) session instead of the spawner's.

    The request-scoped ``HERMES_SESSION_CHAT_ID`` binding survives child
    construction: ``_bind_api_server_session`` binds ``chat_id`` to the raw
    ``X-Hermes-Session-Id``, and its only writer is ``set_session_vars`` —
    ``set_current_session_id`` never touches it. Gate on the platform: on
    push platforms ``chat_id`` is a chat, not a session, so yield ``""``
    there.
    r   get_session_envHERMES_SESSION_PLATFORMrR   
api_serverHERMES_SESSION_CHAT_ID)gateway.session_contextr   rZ   r   s    r   _current_origin_session_idr     sp    $;;;;;;?4b99\II27<<BB   rrs   ( ( 
66rR   )rT   rS   r0   interrupt_fnmax_async_childrenrI   rK   rL   Optional[List[str]]rM   rN   rQ   rT   runnerCallable[[], Dict[str, Any]]rS   r0   r   Optional[Callable[[], None]]r   c                6   t                      t          j                    | ||rt          |          nd|||||	|dd|
d}t          5  t	          d t
                                          D                       }||k    rdd| ddcddd           S |t
          <   ddd           n# 1 swxY w Y   t          |           t          |          }dfd}	 |	                    t          |                     nh# t          $ r[}t          5  t
                              d           ddd           n# 1 swxY w Y   t                     dd| dcY d}~S d}~ww xY wt                              d|pd| pddd                    ddS )a  Spawn ``runner`` on the daemon executor and return a handle immediately.

    Parameters
    ----------
    goal, context, toolsets, role, model
        The dispatch-time task spec, captured verbatim for the rich
        completion block.
    session_key
        The gateway session_key (from ``tools.approval.get_current_session_key``)
        captured on the parent thread BEFORE dispatch, because the daemon
        worker thread won't carry the contextvar. Used to route the
        completion back to the originating session.
    parent_session_id
        The durable ``state.db`` session id of the parent agent that spawned
        the delegation. Carried on the completion event so the gateway can
        pin routing to the spawning session instead of recovering the latest
        ``ended_at IS NULL`` row for the peer tuple (#57498).
    runner
        Zero-arg callable that builds + runs the child and returns the same
        result dict ``_run_single_child`` produces. Runs on the worker thread.
    interrupt_fn
        Optional callable to signal the child to stop (used on shutdown /
        explicit cancel).
    max_async_children
        Concurrency cap. When at capacity the dispatch is REJECTED (the caller
        should fall back to sync or tell the user) rather than queued, so a
        runaway model can't pile up unbounded background work.

    Returns
    -------
    dict
        ``{"status": "dispatched", "delegation_id": ...}`` on success, or
        ``{"status": "rejected", "error": ...}`` when at capacity.
    Nr   )rP   rI   rK   rL   rM   rN   rQ   rS   r0   rT   rt   rU   rv   r   c              3  L   K   | ]}|                     d           dk    dV   dS rt   r   r$   NrE   r   s     r   r   z,dispatch_async_delegation.<locals>.<genexpr>Z  ?       
 
AEE(OOy,H,HA,H,H,H,H
 
r   rejected#Async delegation capacity reached (z running). Wait for one to finish (its result will re-enter the chat), or run this task synchronously (background=false). Raise delegation.max_concurrent_children in config.yaml to allow more concurrent background subagents.rt   r   r   r@   c            	        i } d}	              pi } |                      d          pd}nu# t          $ rh}t                              d           dd t	          |          j         d| dt          t          j                    z
  d          d} d}Y d }~nd }~ww xY wt          | |           d S # t          | |           w xY w)	Nr   rt   ru   zAsync delegation %s crashed: r   r   )rt   r   r   	api_callsduration_seconds)	rF   rZ   r   	exceptionr|   __name__roundrV   	_finalize)rr   rt   excrP   rU   r   s      r   _workerz*dispatch_async_delegation.<locals>._workerm  s    !#	5VXX^FZZ))8[FF 		 		 		:MJJJ! II.77#77$)$)++*Eq$I$I F FFFFFF		 mVV44444ImVV4444s,   #+ B3 
BABB3 BB3 3Cz%Failed to schedule async delegation: z3Dispatched async delegation %s (session_key=%s): %s<cli>rR   P   
dispatchedrt   rP   r   r@   )r   rV   listr   r   r   r   ra   r   submitr   rZ   r   re   r   info)rI   rK   rL   rM   rN   rQ   rT   r   rS   r0   r   r   r>   r   executorr  r  rP   rU   s          `         @@r   dispatch_async_delegationr    s   b '((MIKKM&&.8DNNND" 4..&$ F& 
 ) ) 
 
((
 
 
 
 
 ((($R:L R R R	 	) ) ) ) ) ) ) ) #)) ) ) ) ) ) ) ) ) ) ) ) ) ) )" f/00H5 5 5 5 5 5 5 5&
 	3G<<==== 
 
 
 	. 	.LL---	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	."=111 BSBB
 
 	
 	
 	
 	
 	
 	
	
 KK={-g
CRC/@   #]CCCsZ   ?B0
B00B47B4""D 
E*E%D>2E%>E	E%E	E%E*%E*rt   c                   t           5  t                              |           }|	 ddd           dS d|d<   t          j                    |d<   d|d<   t	          |          }ddd           n# 1 swxY w Y   t          |||           t           5  t                              |           }|||d<   t                       ddd           dS # 1 swxY w Y   dS )zDMark a record complete and push the completion event onto the queue.Nr   rt   rv   r   )r   r   rF   rV   r   _push_completion_eventr   )rP   rr   rt   r>   event_records        r   r  r    sz   	 
$ 
$m,,>
$ 
$ 
$ 
$ 
$ 
$ 
$ 
$ (x!%~!%~F||
$ 
$ 
$ 
$ 
$ 
$ 
$ 
$ 
$ 
$ 
$ 
$ 
$ 
$ 
$ <888	 " "m,,%F8!!!	" " " " " " " " " " " " " " " " " "s(   A./A..A25A20CCCc           	     6   	 ddl m} nG# t          $ r:}t                              d|                     d          |           Y d}~dS d}~ww xY w|                    d          }|                    d          }|                     d          pt          j                    }|                     d	          pt          j                    }i d
dd|                     d          d|                     dd          d|                     dd          d|                     dd          d|                     d          d|                     dd          d|                     d          d|                     d          d|                     d          d|                    d          p|                     d          d|d|d|d|                    dd          d|                    dt          ||z
  d                    d|||                    d          d}	t          |	|           	 |j	        
                    |	           dS # t          $ r:}t                              d|                     d          |           Y d}~dS d}~ww xY w)zPush a type='async_delegation' event onto the shared completion queue.

    Best-effort: a failure here must not crash the worker, but it WOULD mean a
    silently-lost result, so we log loudly.
    r   process_registryzPAsync delegation %s finished but process_registry import failed; result lost: %srP   Nr   r   rU   rv   r|   r}   rQ   rR   rS   r0   rT   rI   rK   rL   rM   rN   rt   r   r   r   exit_reason)rv   r  zHAsync delegation %s: failed to enqueue completion event; result lost: %s)tools.process_registryr  rZ   r   r   rF   rV   r  rw   completion_queuer   )
r>   rr   rt   r  r  r   r   rU   rv   r   s
             r   r  r    s&   ;;;;;;;   JJ''	
 	
 	

 	 jj##GJJwEJJ//>49;;M::n--<L"O44
 	vzz-44 	

+A2 F F 	VZZ(;R@@ 	VZZ(;<< 	

62&& 	6::i(( 	FJJz** 	

6"" 	G$$;

7(;(; 	& 	7  	!" 	VZZQ//#$ 	FJJl]&BA F F
 
%* 	+, %zz-00/  C2 V$$$
)--c22222 
 
 
JJ''	
 	
 	
 	
 	
 	
 	
 	
 	

s,   	 
A/AA8I 
J/JJ)rT   rS   r0   r   r   rP   rJ   	List[str]c                   pt                      t          j                    t          |           }|dk    r| d         n$| dd                    d | D                       z   }i dd|dt	          |           d	|d
|rt	          |          ndd|d|d|d|d|	d|dddddd|
dd}t
          5  t          d t                                          D                       }||k    rdd| ddcddd           S |t          <   ddd           n# 1 swxY w Y   t          |           t          |          }d&fd }	 |                    t          |                     nh# t          $ r[}t
          5  t                              d           ddd           n# 1 swxY w Y   t                     dd!| dcY d}~S d}~ww xY wt                               d"||pd#           d$d%S )'u+  Dispatch a WHOLE fan-out batch as ONE background unit.

    Unlike ``dispatch_async_delegation`` (which backs a single subagent),
    ``runner`` here runs the entire batch — it builds and joins on every child
    in parallel and returns the combined ``{"results": [...],
    "total_duration_seconds": N}`` dict that the synchronous path would have
    returned. We occupy ONE async slot for the whole batch (the in-batch
    parallelism is bounded separately by ``max_concurrent_children``), so a
    single ``delegate_task`` fan-out never exhausts the async pool by itself.

    When the batch finishes, a SINGLE completion event is pushed onto the
    shared ``process_registry.completion_queue`` carrying the full per-task
    ``results`` list, so the consolidated summaries re-enter the conversation
    as one message once every child is done — the chat is never blocked while
    they run.

    Returns ``{"status": "dispatched", "delegation_id": ...}`` on success or
    ``{"status": "rejected", "error": ...}`` when the async pool is at
    capacity.
    r$   r   z parallel subagents: z; c              3  *   K   | ]}|d d         V  d S )N(   r   )r%   gs     r   r   z2dispatch_async_delegation_batch.<locals>.<genexpr>  s+      G^G^ST#2#G^G^G^G^G^G^r   rP   rI   rJ   rK   rL   NrM   rN   rQ   rS   r0   rT   rt   r   rU   rv   r   rO   Tc              3  L   K   | ]}|                     d           dk    dV   dS r   rE   r   s     r   r   z2dispatch_async_delegation_batch.<locals>.<genexpr>#  r   r   r   r   z running). Wait for one to finish (its result will re-enter the chat), or raise delegation.max_concurrent_children in config.yaml to allow more concurrent background units.r   r   r@   c                    i } d}	              pi } |                      d          pg }|rt          d |D                       rd}nd}ns# t          $ rf}t                              d           g t          |          j         d| t          t          j                    z
  d          d} d}Y d }~nd }~ww xY wt          | |           d S # t          | |           w xY w)	Nr   resultsc              3  D   K   | ]}|                     d           dvV  dS )rt   )ru   successNrE   r   s     r   r   zCdispatch_async_delegation_batch.<locals>._worker.<locals>.<genexpr><  sF       % % x(@@% % % % % %r   ru   z!Async delegation batch %s crashedr   r   )r   r   total_duration_seconds)
rF   allrZ   r   r   r|   r  r  rV   _finalize_batch)combinedrt   child_resultsr  rP   rU   r   s       r   r  z0dispatch_async_delegation_batch.<locals>._worker5  s5   #%	=vxx~2H$LL339rM % % %&% % % " " % !$ 	 	 	@-PPP II.77#77*/	m0KQ*O*O H
 FFFFFF	 M8V<<<<<OM8V<<<<s1   AA 
C 
B;AB61C 6B;;C C$z+Failed to schedule async delegation batch: zADispatched async delegation batch %s (%d task(s), session_key=%s)r  r  r	  r
  )r   rV   r   joinr  r   r   r   r   ra   r   r  r   rZ   r   re   r   r  )rJ   rK   rL   rM   rN   rQ   rT   r   rS   r0   r   r   rP   ncombined_goalr>   r   r  r  r  rU   s          `    `       @r   dispatch_async_delegation_batchr+    s   H "9%7%9%9MIKKME

A FFa1 ; ; ;diiG^G^X]G^G^G^>^>^ ^  	e 	7	
 	h8DNNND 	 	 	{ 	 4 	. 	. 	) 	 	 	  	D!F$ 
 ) ) 
 
((
 
 
 
 
 ((($N:L N N N ) ) ) ) ) ) ) ) #)) ) ) ) ) ) ) ) ) ) ) ) ) ) )  f/00H= = = = = = = =2

3G<<==== 
 
 
 	. 	.LL---	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	. 	."=111 H3HH
 
 	
 	
 	
 	
 	
 	
	
 KKKq+0   #]CCCsZ    ?D"
D""D&)D&"E7 7
GGF0$G0F4	4G7F4	8GGGr&  c                   t           5  t                              |           }|	 ddd           dS d|d<   t          j                    |d<   d|d<   t	          |          }ddd           n# 1 swxY w Y   	 ddlm} n4# t          $ r'}t          	                    d| |           Y d}~dS d}~ww xY w|                    d	          pt          j                    }|                    d          pt          j                    }i d
dd| d|                    dd          d|                    dd          d|                    dd          d|                    d          d|                    dd          d|                    d          d|                    d          d|                    d          d|                    d          d|                    d          d|ddd|                    d          pg d|                    d          d|                    d          |                    d          ||d}	t          |	|           	 |j                            |	           n3# t          $ r&}t          	                    d| |           Y d}~nd}~ww xY wt           5  t                              |           }|||d<   t                       ddd           dS # 1 swxY w Y   dS # t           5  t                              |           }|||d<   t                       ddd           w # 1 swxY w Y   w xY w) zDMark a batch record complete and push ONE combined completion event.Nr   rt   rv   r   r   r  zVAsync delegation batch %s finished but process_registry import failed; result lost: %srU   r|   r}   rP   rQ   rR   rS   r0   rT   rI   rJ   rK   rL   rM   rN   rO   Tr   live_transcriptsr   r#  )r#  rU   rv   zNAsync delegation batch %s: failed to enqueue completion event; result lost: %s)r   r   rF   rV   r   r  r  rZ   r   r   rw   r  r   r   )
rP   r&  rt   r>   r  r  r  rU   rv   r   s
             r   r%  r%  a  s    
 $ $m,,>$ $ $ $ $ $ $ $ (x!%~!%~F||$ $ $ $ $ $ $ $ $ $ $ $ $ $ $;;;;;;;   &3	
 	
 	

 	 !$$_55DM##N33Bty{{L" 	|''r:: 	 0 01G L L	
 	\--.A2FF 	\--.ABB 	  ,, 	!!'** 	<##I.. 	L$$Z00 	  (( 	!!'** 	& 	D" 	8<<	**0b#* 	HLL);<<+, 	g&&-. #+,,/G"H"H&$3  C6 X&&&&)--c2222 
 
 
3	
 	
 	
 	
 	
 	
 	
 	

  	& 	&\\-00F!#)x #%%%		& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	&] 	& 	&\\-00F!#)x #%%%		& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	& 	&s   A./A..A25A2:B 
B2B--B2!I< ;L  <
J,J'"L  'J,,L  60K33K7:K7 M0M8MMMMMList[Dict[str, Any]]c                     t           5  d t                                          D             cddd           S # 1 swxY w Y   dS )zSnapshot of async delegations (running + recently completed).

    Safe to call from any thread. Excludes the non-serialisable interrupt_fn.
    c                J    g | ] }d  |                                 D             !S )c                &    i | ]\  }}|d k    ||S )r   r   )r%   kvs      r   rH   z5list_async_delegations.<locals>.<listcomp>.<dictcomp>  s(    ???da1+>+>Q+>+>+>r   )r   r   s     r   r   z*list_async_delegations.<locals>.<listcomp>  s?     
 
 
 @?aggii???
 
 
r   N)r   r   r   r   r   r   list_async_delegationsr4    s    
 
 
 

 
__&&
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s   #8<<shutdownreasonc                   d}t           5  d t                                          D             }ddd           n# 1 swxY w Y   |D ]}}|                    d          }t	          |          rW	  |             |dz  }7# t
          $ r9}t                              d|                    d          |           Y d}~ud}~ww xY w~|rt                              d||            |S )	a*  Signal every running async delegation to stop. Returns how many.

    Used on ``/stop`` and gateway shutdown so a dangling background subagent
    can't keep burning tokens with no one listening. The child still emits a
    completion event (status='interrupted') via the normal finalize path.
    r   c                D    g | ]}|                     d           dk    |S r   rE   r   s     r   r   z!interrupt_all.<locals>.<listcomp>  s3     
 
 
AEE(OOy,H,HA,H,H,Hr   Nr   r$   z&interrupt_all: %s interrupt failed: %srP   z'Interrupted %d async delegation(s) (%s)	r   r   r   rF   callablerZ   r   debugr  )r6  counttargetsr   fnr  s         r   interrupt_allr?    se    E	 
 

 
((
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
  
 
UU>""B<< 	
   <EE/**C       		  N=ufMMMLs%   $:>>.A>>
C/B<<Csession_endc                     sssdS d}t           5   fdt                                          D             }ddd           n# 1 swxY w Y   |D ]}}|                    d          }t	          |          rW	  |             |dz  }7# t
          $ r9}t                              d|                    d          |           Y d}~ud}~ww xY w~|rt                              d||           |S )	u{  Signal running async delegations owned by ONE session to stop.

    A delegation's lifecycle is bound to the session that spawned it: when
    that session ends, its in-flight background subagents must end with it —
    a completed orphan would otherwise sit on the shared completion queue
    with no live owner, either leaking into another chat or burning tokens
    with no one listening (#55578).

    Selectors (any matching field claims the record):
    - ``origin_ui_session_id``: the live TUI tab/window that commissioned it.
    - ``session_key``: the durable routing key captured at dispatch.
    - ``parent_session_id``: the spawning agent's durable session-db id —
      the right selector for gateway chats, whose ``session_key`` (the
      platform conversation key) SURVIVES a ``/new`` reset while the
      session id rotates.

    Returns how many were interrupted.
    r   c                B   g | ]}|                     d           dk    rr(t          |                     d          pd          k    sTr(t          |                     d          pd          k    s*r*t          |                     d          pd          k    |S )rt   r   rS   rR   rQ   rT   )rF   rb   )r%   r   rS   rT   rQ   s     r   r   z)interrupt_for_session.<locals>.<listcomp>  s     
 
 
uuX)++% ,*-aee4J.K.K.Qr*R*RVj*j*j +k$'m(<(<(B$C$C{$R$R% %S*-aee4G.H.H.NB*O*OSd*d*d 
 +e*d*dr   Nr   r$   z.interrupt_for_session: %s interrupt failed: %srP   z:Interrupted %d async delegation(s) for ending session (%s)r9  )	rQ   rS   rT   r6  r<  r=  r   r>  r  s	   ```      r   interrupt_for_sessionrC    s   0  3 <M qE	 	
 	

 
 
 
 
 
((
 
 
	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
  
 
UU>""B<< 	
   DEE/**C       		  
H6	
 	
 	
 Ls)   (A		AA=B
C/CCc                    t           5  t          t                              d           dadaddd           n# 1 swxY w Y   t          5  t
                                           ddd           dS # 1 swxY w Y   dS )z6Test-only: clear all state and tear down the executor.NF)waitr   )r   r   r5  r   r   r   clearr   r   r   _reset_for_testsrG    s    
 " " E***	 !	" " " " " " " " " " " " " " "
 
                   s   ';??A44A8;A8)r   r   )r>   r?   r   r@   )rP   rb   r   r@   r
  )rq   r?   rr   r?   r   r@   )r   r   )rP   rb   r   r   )rP   rb   r   rb   r   r   )r   r?   r   rb   r   r   )r   r?   r   rb   r   r@   )rP   rb   r   r   )r   r   r   r   )r   rb   )rI   rb   rK   r   rL   r   rM   rb   rN   r   rQ   rb   rT   r   r   r   rS   rb   r0   rb   r   r   r   r   r   r?   )rP   rb   rr   r?   rt   rb   r   r@   )r>   r?   rr   r?   rt   rb   r   r@   )rJ   r  rK   r   rL   r   rM   rb   rN   r   rQ   rb   rT   r   r   r   rS   rb   r0   rb   r   r   r   r   rP   r   r   r?   )rP   rb   r&  r?   rt   rb   r   r@   )r   r.  )r5  )r6  rb   r   r   )rR   rR   rR   r@  )
rQ   rb   rS   rb   rT   rb   r6  rb   r   r   )H__doc__
__future__r   r\   loggingr5   	threadingrV   r   concurrent.futuresr   typingr   r   r   r   r	   hermes_constantsr   tools.daemon_poolr   tools.thread_contextr   	getLoggerr  r   r   r   __annotations__Lockr   r   r   r   _DEFAULT_MAX_ASYNC_CHILDRENrj   rg   rk   r   r[   r   r=   ra   re   r^   rw   ry   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r+  r%  r4  r?  rC  rG  r   r   r   <module>rU     s  ! ! !F # " " " " "          1 1 1 1 1 1 6 6 6 6 6 6 6 6 6 6 6 6 6 6 , , , , , , 6 6 6 6 6 6 < < < < < <		8	$	$
 5  +/	 . . . .!!     	   ') ( ( ( (  -  
  9>* * *- - - -`   :^ ^ ^ ^
$ $ $ $N	
 	
 	
 	

 
 
 
0 0 0 0f   6	! 	! 	! 	!! ! ! !(T T T T!! !! !! !!H! ! ! !,! ! ! !T T T T
S S S S
   &   &a a a a+ + + +       $   H (, "159~D ~D ~D ~D ~D ~DB" " " ",8
 8
 8
 8
F (, "159#'yD yD yD yD yD yDxB& B& B& B&J	
 	
 	
 	
    :  "	5 5 5 5 5p	 	 	 	 	 	r   