o iq@s&ddlmZddlZddlZddlZddlZddlZddlZddlm Z m Z ddlm Z ddl m Z ddlmZmZmZmZmZmZddlmZdd lmZdd lmZmZdd lmZmZdd lm Z m!Z!dd lm"Z"ddl#m$Z$erzddl%m&Z&e'e(ddZ)edZ*e GdddZ+e GdddZ,e GdddZ-e GdddZ.e+e,Be-Be.BZ/Gdddee*Z0GdddZ1Gdd d Z2Gd!d"d"Z3Gd#d$d$Z4Gd%d&d&Z5Gd'd(d(Z6Gd)d*d*Z7ere8e9e&e8e:e ffZ;ejdd5d>d;d<Z?dS)?) annotationsN)Callable Generator)contextmanager) dataclass) TYPE_CHECKINGAnyGenericLiteralTypeVaroverload)trace)llm) function_toolutils) trace_typestracer) NOT_GIVEN NotGivenOr)is_given) SpeechHandle)AgentLIVEKIT_EVALS_VERBOSERun_Tc@"eZdZUded<dZded<dS)ChatMessageEventzllm.ChatMessageitemmessageLiteral['message']typeN__name__ __module__ __qualname____annotations__r!r'r'a/var/www/html/livekit_bhavya/venv/lib/python3.10/site-packages/livekit/agents/voice/run_result.pyr' rc@r)FunctionCallEventzllm.FunctionCallr function_callLiteral['function_call']r!Nr"r'r'r'r(r*-r)r*c@r)FunctionCallOutputEventzllm.FunctionCallOutputrfunction_call_outputLiteral['function_call_output']r!Nr"r'r'r'r(r-3r)r-c@s2eZdZUded<ded<ded<dZded <d S) AgentHandoffEventllm.AgentHandoffr Agent | None old_agentr new_agent agent_handoffLiteral['agent_handoff']r!Nr"r'r'r'r(r09s r0c@seZdZddd5d d Zed6d d Zejd7ddZed8ddZ d9ddZ d:ddZ d;d d!Z dd,d-Zd?d.d/Zd@d3d4ZdS)A RunResultN) user_inputr8 str | None output_typetype[Run_T] | NonereturnNonecCs8t|_tjd|_||_||_g|_d|_d|_ dSN) set_handlesasyncioFuture _done_fut _user_input _output_type_recorded_items _final_output_RunResult__last_speech_handle)selfr8r:r'r'r(__init__Es zRunResult.__init__list[RunEvent]cC|jS)z5List of all recorded events generated during the run.)rFrIr'r'r(eventsPszRunResult.events RunAssertcCs4trdt|j}td|jd|dt|S)z Provides an assertion helper for verifying the run events. Returns: RunAssert: Assertion interface for run events. z z + RunResult( user_input=`z` events: z ))lk_evals_verbosejoin_format_eventsrNprintrDrO)rI events_strr'r'r(expectUs zRunResult.expectrcCs&|js td|jstd|jS)z Returns the final output of the run after completion. Raises: RuntimeError: If the run is not complete or no output is set. Returns: Run_T: The final result output. z3cannot retrieve final_output, RunResult is not donezno final output)rCdone RuntimeErrorrGrMr'r'r( final_outputis zRunResult.final_outputboolcCs |jS)z=Indicates whether the run has finished processing all events.)rCrVrMr'r'r(rV|s zRunResult.done'Generator[None, None, RunResult[Run_T]]csdfdd }|S)Nr<RunResult[Run_T]cstjIdHSr>)rAshieldrCr'rMr'r( _await_implsz(RunResult.__await__.._await_impl)r<r[) __await__)rIr]r'rMr(r^s zRunResult.__await__rr1r3r2r4rcCs0t|||d}|j|jjd}|j||dS)N)rr3r4 created_at)r0_find_insertion_indexrr`rFinsert)rIrr3r4eventindexr'r'r(_agent_handoffszRunResult._agent_handoff llm.ChatItemcCs|jrdSd}|jdkrt|d}n|jdkrt|d}n |jdkr)t|d}|dur>|j|jjd}|j ||dSdS)Nr)rr+r.r_) rCrVr!rr*r-rarr`rFrb)rIrrcrdr'r'r( _item_addeds       zRunResult._item_addedhandleSpeechHandle | asyncio.TaskcCs2|j|t|tr||j||jdSr>)r@add isinstancer_add_item_added_callbackrgadd_done_callback_mark_done_if_neededrIrhr'r'r( _watch_handles   zRunResult._watch_handlecCs6|j|||jt|tr||jdSdSr>)r@discardremove_done_callbackrnrkr_remove_item_added_callbackrgror'r'r(_unwatch_handles   zRunResult._unwatch_handle"SpeechHandle | asyncio.Task | NonecCs4t|tr||_tdd|jDr|dSdS)Ncss|]}|VqdSr>)rV).0rhr'r'r( sz1RunResult._mark_done_if_needed..)rkrrHallr@ _mark_doneror'r'r(rns  zRunResult._mark_done_if_neededc Csttjd|jdur|jd WddS|jj}t|t sL|j rBt||j sB|j t d|j j dt|jj n ||_|jdn|j |WddSWddSWddS1smwYdS)NzExpected output of type , got ) contextlibsuppressrAInvalidStateErrorrHrC set_result_maybe_run_final_outputrk BaseExceptionrE set_exceptionrWr#r!rG)rIrXr'r'r(rys.     "zRunResult._mark_doner`floatintcCs:ttt|jD]}|j|jj|kr|dSq dS)z Returns the index to insert an item by creation time. Iterates in reverse, assuming items are sorted by `created_at`. Finds the position after the last item with `created_at <=` the given timestamp. rr)reversedrangelenrFrr`)rIr`ir'r'r(ras  zRunResult._find_insertion_index)r8r9r:r;r<r=)r<rK)r<rO)r<r)r<rY)r<rZ)rr1r3r2r4rr<r=)rrfr<r=)rhrir<r=)rhrur<r=r<r=)r`rr<r)r#r$r%rJpropertyrN functoolscached_propertyrUrXrVr^rergrprtrnryrar'r'r'r(r7Ds"           r7c@sveZdZdYddZedZd d Zed[dd Zd\dd Zd]ddZd^d_ddZeddd`ddZedad"dZedbd%dZedcd(dZeddd+dZddded.dZee d/dfd3d4Z ee e d5dgd;d4Z ee e d<dhdAd4Z ee dBdidFd4Z e e e e e e dGdjdJd4Z dkdldMdNZ dmdOdPZ e e d5dndQdRZ e d/dodSdTZe e d<dpdUdVZe dBdqdWdXZdS)rrO run_resultr7cCs|j|_d|_dS)Nr)rN _events_list_current_index)rIrr'r'r(rJs zRunAssert.__init__rdrr< EventAssertcCdSr>r')rIrdr'r'r( __getitem__zRunAssert.__getitem__ssliceEventRangeAssertcCrr>r')rIrr'r'r(rrkey [int, slice]EventAssert | EventRangeAssertcCst|tr|j|}t|||St|trH|dkr |t|j7}d|kr-t|jks?n|jd|dt|jd|dt|j|||Stt |j dt |j )a, Access a specific event or range for assertions. Args: key (int | slice): Index or slice of events. Returns: EventAssert: Assertion for a single event when key is int. EventRangeAssert: Assertion for a span of events when key is slice. Raises: TypeError: If key is not an int or slice. AssertionError: If index is out of range. Examples: # Single event access >>> result.expect[0].is_message(role="user") >>> result.expect[-1].is_message(role="assistant") # Full range access >>> result.expect[:].contains_function_call(name="foo") # Partial range access >>> result.expect[0:2].contains_message(role="assistant") rznth(z) out of range (total events: )rdz# indices must be int or slice, not ) rkrrrrr_raise_with_debug_infor TypeErrorr!r#)rIrrNr'r'r(rs    cCs,d}|jt|jkr|d||j}|S)NTz&Expected another event, but none left.)rrrrrI__tracebackhide__rcr'r'r(_current_event s   zRunAssert._current_eventNrstr int | Noner=cCs<d}|dur |jn|}dt|j|d}t|d|)NT selected_indexz Context around failure: )rrQrRrAssertionError)rIrrdr marker_indexrTr'r'r(rsz RunAssert._raise_with_debug_info)r!r!cCrr>r'rIr!r'r'r( next_eventrzRunAssert.next_eventr ChatMessageAssertcCrr>r'rr'r'r(r rr,FunctionCallAssertcCrr>r'rr'r'r(r#rr/FunctionCallOutputAssertcCrr>r'rr'r'r(r&rr6AgentHandoffAssertcCrr>r'rr'r'r(r)rSLiteral['message', 'function_call', 'function_call_output', 'agent_handoff'] | NonedEventAssert | ChatMessageAssert | FunctionCallAssert | FunctionCallOutputAssert | AgentHandoffAssertcCszd} |}|jd7_|dus|j|krnq|dkr#|S|dkr+|S|dkr3|S|dkr;|S|S)a] Advance to the next event, optionally filtering by type. Args: type (str, optional): Event type to match. Returns: EventAssert or subclass: Assertion object for the matched event. Example: >>> result.expect.next_event(type="function_call").is_function_call(name="foo") TrNrr+r.r5)rrrcr! is_messageis_function_callis_function_call_outputis_agent_handoff)rIr!r ev_assertr'r'r(r,s rolerNotGivenOr[llm.ChatRole]ChatMessageAssert | NonecCrr>r')rIr!rr'r'r(skip_next_event_ifXzRunAssert.skip_next_event_ifname argumentsrNotGivenOr[str]rNotGivenOr[dict[str, Any]]FunctionCallAssert | NonecCrr>r')rIr!rrr'r'r(r]outputis_errorrrNotGivenOr[bool]FunctionCallOutputAssert | NonecCrr>r')rIr!rrr'r'r(rfrnew_agent_typerNotGivenOr[type[Agent]]AgentHandoffAssert | NonecCrr>r')rIr!rr'r'r(ror)rrrrrrLLiteral['message', 'function_call', 'function_call_output', 'agent_handoff']]ChatMessageAssert | AgentHandoffAssert | FunctionCallAssert | FunctionCallOutputAssert | Nonec Csd}zAd} |dkr|j|d} n(|dkr |j||d} n|dkr.|j||d} n |d kr:|j|d } |jd 7_| WStyMYdSw) a Conditionally skip the next event if it matches criteria. Args: type (str): Type of event to check. role (ChatRole, optional): Required role for message events. name (str, optional): Required function name for calls. arguments (dict, optional): Required args for function calls. output (str, optional): Required output for function call outputs. is_error (bool, optional): Required error flag for call outputs. new_agent_type (type, optional): Required agent class for handoffs. Returns: EventAssert or None: The skipped event assertion if matched. Example: >>> skipped = result.expect.skip_next_event_if(type="message", role="assistant") TNrrr+rr.rr5rr)rrrrrrrrW) rIr!rrrrrrrevr'r'r(rts"# rcountcCsLd}t|D]}|jt|jkr|d|d|d|jd7_q|S)a  Skip a specified number of upcoming events without assertions. Args: count (int): Number of events to skip. Returns: RunAssert: Self for chaining. Example: >>> result.expect.skip_next(2) TzTried to skip z event(s), but only z were available.r)rrrrr)rIrrrr'r'r( skip_nexts zRunAssert.skip_nextcCs>d}|jt|jkr|j|j}|dt|jdSdS)z Assert that there are no further events. Raises: AssertionError: If unexpected events remain. Example: >>> result.expect.no_more_events() Tz$Expected no more events, but found: N)rrrrr!r#rr'r'r(no_more_eventss  zRunAssert.no_more_eventscCd}|ddj||dS)ay Assert existence of a function call event matching criteria. Args: name (str, optional): Function name to match. arguments (dict, optional): Arguments to match. Returns: FunctionCallAssert: Assertion for the matching call. Example: >>> result.expect.contains_function_call(name="foo") TNr)contains_function_call)rIrrrr'r'r(rz RunAssert.contains_function_callcCd}|ddj|dS)a0 Assert existence of a message event matching criteria. Args: role (ChatRole, optional): Role to match. Returns: ChatMessageAssert: Assertion for the matching message. Example: >>> result.expect.contains_message(role="user") TNr)contains_messagerIrrr'r'r(rszRunAssert.contains_messagecCr)a Assert existence of a function call output event matching criteria. Args: output (str, optional): Output string to match. is_error (bool, optional): Error flag to match. Returns: FunctionCallOutputAssert: Assertion for the matching output. Example: >>> result.expect.contains_function_call_output(is_error=True) TNr)contains_function_call_outputrIrrrr'r'r(rrz'RunAssert.contains_function_call_outputcCr)aZ Assert existence of an agent handoff event matching criteria. Args: new_agent_type (type, optional): Expected new agent class. Returns: AgentHandoffAssert: Assertion for the matching handoff. Example: >>> result.expect.contains_agent_handoff(new_agent_type=MyAgent) TNr)contains_agent_handoffrIrrr'r'r(rsz RunAssert.contains_agent_handoff)rr7)rdrr<r)rrr<r)rrr<r)r<rr>)rrrdrr<r=)r!r=r<r)r!r r<r)r!r,r<r)r!r/r<r)r!r6r<r)r!rr<r)r!r rrr<r)r!r,rrrrr<r)r!r/rrrrr<r)r!r6rrr<r)r!rrrrrrrrrrrrrr<r)r)rrr<rOrrrrrr<rrrr<rrrrrr<rrrr<r)r#r$r%rJr rrrrrrrrrrrrr'r'r'r(rOsl     ,     , < rOc@speZdZd.d/dd Zd0ddZd1ddZeedd2ddZeedd3dd Zed!d4d%d&Z ed'd5d+d,Z d-S)6rrcRunEventparentrOrdrcC||_||_||_dSr>_event_parent_indexrIrcrrdr'r'r(rJ. zEventAssert.__init__rrr<r=cCd}|jj||jddSNTrrrrrIrrr'r'r(_raise3zEventAssert._raisecCrLr>rrMr'r'r(rc7zEventAssert.eventrrrrrrc Csd}t|jts |dt|jtsJt|r/|jjj|kr/|d|d|jjjdt|r_t|jjj }| D]\}}||vsM|||kr^|d|d|d| |q?t |j|j |jS) a Verify this event is a function call with matching details. Args: name (str, optional): Expected function name. arguments (dict, optional): Expected call arguments. Returns: FunctionCallAssert: Assertion for the function call. Raises: AssertionError: If the event is not a function call or details mismatch. Example: >>> ev_assert.is_function_call(name="foo", arguments={"x": 1}) TzExpected FunctionCallEventzExpected call name '', got ''z For key 'z ', expected rz)rkrr*rrrrjsonloadsritemsgetrrr)rIrrractualrvaluer'r'r(r:s  "zEventAssert.is_function_callrrrrrcCsd}t|jts |dt|jtsJt|r/|jjj|kr/|d|d|jjjdt|rH|jjj|krH|d|d|jjjt|j|j |j S)a Verify this event is a function call output with matching details. Args: output (str, optional): Expected output text. is_error (bool, optional): Expected error flag. Returns: FunctionCallOutputAssert: Assertion for the output. Raises: AssertionError: If the event is not function output or details mismatch. Example: >>> ev_assert.is_function_call_output(output="OK", is_error=False) Tz Expected FunctionCallOutputEventzExpected output 'rrzExpected is_error=rz) rkrr-rrrrrrrrrr'r'r(ras  z#EventAssert.is_function_call_outputrrrrcCspd}t|jts |dt|jtsJt|r/|jjj|kr/|d|d|jjjdt|j|j|j S)a Verify this event is a message from the given role. Args: role (ChatRole, optional): Expected sender role. Returns: ChatMessageAssert: Assertion for the message. Raises: AssertionError: If the event is not a message or role mismatch. Example: >>> ev_assert.is_message(role="assistant") TzExpected ChatMessageEventzExpected role 'rr) rkrrrrrrrrrrr'r'r(rs  zEventAssert.is_messagerrrrcCsvd}t|jts |dt|jtsJt|r2t|jj|s2|d|jdt|jjjdt|j|j |j S)a Verify this event is an agent handoff. Args: new_agent_type (type, optional): Expected new agent class. Returns: AgentHandoffAssert: Assertion for the handoff. Raises: AssertionError: If the event is not an agent handoff or type mismatch. Example: >>> ev_assert.is_agent_handoff(new_agent_type=MyAgent) TzExpected AgentHandoffEventzExpected new_agent 'rr) rkrr0rrr4r#r!rrrrr'r'r(rs  zEventAssert.is_agent_handoffN)r)rcrrrOrdrrrr<r=)r<rrrrr) r#r$r%rJrrcrrrrrr'r'r'r(r-s   ( rc@sZeZdZd&ddZeed d'ddZedd(ddZeedd)ddZedd*d#d$Zd%S)+rrNrKrrOrngrcCrr>)_eventsr_rng)rIrNrrr'r'r(rJrzEventRangeAssert.__init__rrrrrr<rc Cd}t|jD]/\}}t||j|jjpd|}tt|j ||dWdS1s1wYq|j d|jt d)a Assert that a function call matching criteria exists in the event range. Args: name (str, optional): Expected function name. arguments (dict, optional): Expected call arguments. Returns: FunctionCallAssert: Assertion for the matched function call. Raises: AssertionError: If no matching function call is found in range. Example: >>> result.expect[0:3].contains_function_call(name="foo") TrrNz8No FunctionCallEvent satisfying criteria found in range unreachable) enumeraterrrrstartr{r|rrrrW)rIrrridxr candidater'r'r(r  & z'EventRangeAssert.contains_function_callrrrrc Cd}t|jD].\}}t||j|jjpd|}tt|j |dWdS1s0wYq|j d|jt d)a Assert that a message matching criteria exists in the event range. Args: role (ChatRole, optional): Expected sender role. Returns: ChatMessageAssert: Assertion for the matched message. Raises: AssertionError: If no matching message is found in range. Example: >>> result.expect[:2].contains_message(role="assistant") TrrNz5No ChatMessageEvent matching criteria found in range r) rrrrrrr{r|rrrrW)rIrrrrrr'r'r(rs  & z!EventRangeAssert.contains_messagerrrrrc Cr)a Assert that a function call output matching criteria exists in the event range. Args: output (str, optional): Expected output text. is_error (bool, optional): Expected error flag. Returns: FunctionCallOutputAssert: Assertion for the matched output. Raises: AssertionError: If no matching output is found in range. Example: >>> result.expect[1:4].contains_function_call_output(is_error=True) TrrNz>> result.expect[0:3].contains_agent_handoff(new_agent_type=MyAgent) TrrNz6No AgentHandoffEvent matching criteria found in range r) rrrrrrr{r|rrrrW)rIrrrrrr'r'r(r&s  & z'EventRangeAssert.contains_agent_handoffN)rNrKrrOrrrrrr) r#r$r%rJrrrrrr'r'r'r(rs %##rc@s>eZdZdddZdd dZdddZeddddZdS)rrcrrrOrdrcCrr>rrr'r'r(rJFrzChatMessageAssert.__init__rrr<r=cCrrrrr'r'r(rKrzChatMessageAssert._raisecCrLr>rrMr'r'r(rcOrzChatMessageAssert.eventjudge_evaluationllm_vllm.LLMintentc sNd}t}|jjj}|tjd|tjj |tj d|tj t ||d|s:|dtd|sE|dtdtd.dd}t}|jddd|jdd|d|dd}d} i} dg} tfdd| Ds|d| d<j||gdddid| d 2z3dHW} | jdur| j} | jsq| jjr| jjd!} | j}q6|s|d"t|tsJtj||d#\}}||i|IdH\}}|tj| |tj|| r| tj!| j"tj#| j$tj%| j"tj&| j$tj'| j(i|s|d$||St)r%d!d%l*m+}||,d&d'd(d)d*}t-d+|d,|d-|S)/a Evaluate whether the message fulfills the given intent. Args: llm_v (llm.LLM): LLM instance for judgment. intent (str): Description of the expected intent. Returns: ChatMessageAssert: Self for chaining further assertions. Example: >>> await msg_assert.judge(llm, intent="should ask for size") Tjudger)rrzThe chat message is empty.rz(Intent is required to judge the message.successrYreasonrr<tuple[bool, str]cs ||fS)z Determines whether the message correctly fulfills the given intent. Args: success: Whether the message satisfies the intent. reason: A concise explanation justifying the result. r')r r r'r'r( check_intentvs z-ChatMessageAssert.judge..check_intentsystemanYou are a test evaluator for conversational agents. You will be shown a message and a target intent. Determine whether the message accomplishes the intent. Only respond by calling the `check_intent(success: bool, reason: str)` function with your final judgment. Be strict: if the message does not clearly fulfill the intent, return `success = False` and explain why.)rcontentuserzCCheck if the following message fulfills the given intent. Intent: z Message: Nzgpt-5c3s|]}|jvVqdSr>)model)rvexcluded_modelrr'r(rwsz*ChatMessageAssert.judge..g temperaturefunctionrr )r!r)chat_ctxtools tool_choice extra_kwargsrz0LLM did not return any arguments for evaluation.)fncjson_argumentszJudgement failed: )shortenrz\nz...)width placeholderz- Judgment succeeded for `z`: ``)r rYr rr<r ).r get_current_spanrr text_content set_attributerATTR_GEN_AI_OPERATION_NAMEATTR_GEN_AI_REQUEST_MODELrATTR_FUNCTION_TOOL_NAMEATTR_FUNCTION_TOOL_ARGSrdumpsrrWrr ChatContext add_messageanychatusagedelta tool_callsrrkr llm_utilsprepare_function_argumentsATTR_FUNCTION_TOOL_IS_ERRORATTR_FUNCTION_TOOL_OUTPUTset_attributesATTR_GEN_AI_USAGE_INPUT_TOKENS prompt_tokensATTR_GEN_AI_USAGE_OUTPUT_TOKENScompletion_tokens#ATTR_GEN_AI_USAGE_INPUT_TEXT_TOKENS$ATTR_GEN_AI_USAGE_OUTPUT_TEXT_TOKENS%ATTR_GEN_AI_USAGE_INPUT_CACHED_TOKENSprompt_cached_tokensrPtextwraprreplacerS)rIrrr current_span msg_contentr rrr,rexcluded_models_temperaturechunktoolfnc_args fnc_kwargsr r r print_msgr'rr(rRs              zChatMessageAssert.judgeN)rcrrrOrdrr)r<r)rrrrr<r) r#r$r%rJrrcrstart_as_current_spanrr'r'r'r(rEs    rc@ eZdZd ddZdd d Zd S)rrcr*rrOrdrcCrr>rrr'r'r(rJrzFunctionCallAssert.__init__r<cCrLr>rrMr'r'r(rcrzFunctionCallAssert.eventN)rcr*rrOrdr)r<r*r#r$r%rJrcr'r'r'r(r rc@rG)rrcr-rrOrdrcCrr>rrr'r'r(rJrz!FunctionCallOutputAssert.__init__r<cCrLr>rrMr'r'r(rcrzFunctionCallOutputAssert.eventN)rcr-rrOrdr)r<r-rHr'r'r'r(rrIrc@rG)rrcr0rrOrdrcCrr>rrr'r'r(rJrzAgentHandoffAssert.__init__r<cCrLr>rrMr'r'r(rcrzAgentHandoffAssert.eventN)rcr0rrOrdr)r<r0rHr'r'r'r(rrIr MockToolsagents_mock_toolsagent type[Agent]mocksdict[str, Callable]r<Generator[None, None, None]c csJti}i|||i}t|}z dVWt|dSt|w)a Temporarily assign a set of mock tool callables to a specific Agent type within the current context. Usage: with mock_tools(MyAgentClass, {"tool_name": mock_fn}): # inside this block, MyAgentClass will see the given mocks N)_MockToolsContextVarrr?reset)rLrNcurrentupdatedtokenr'r'r( mock_toolss  rVrrNrKrr list[str]c Csg}t|D][\}}d}|dur||krdnd}t|tttfr<|jjddhdd}|d|d|jjd |d }n t|t rR|d|d |j d |j d }n |d|d|}| |q|S) Nz>>>z T>idr!call_idr`) exclude_noneexclude_defaultsexcludez [z] z(item=rz] AgentHandoffEvent(old_agent=z , new_agent=) rrkrr*r-r model_dump __class__r#r0r3r4append)rNrlinesrrcprefix item_reprliner'r'r(rRs,"   rR)rLrMrNrOr<rP)rNrKrrr<rW)@ __future__rrAr{ contextvarsrroscollections.abcrrr dataclassesrtypingrrr r r r opentelemetryr rXrrrr/ telemetryrrtypesrrr speech_handlerrLrrgetenvrPrrr*r-r0rr7rOrrrrrrdictr!rrJ ContextVarrQrVrRr'r'r'r(sf        X