o
    iD                     @  s   d dl mZ d dlZd dlmZ ddlmZmZ ddlm	Z	m
Z
mZ eegee eeeeef  B f ZG dd	 d	ZG d
d dee	ZG dd deeZdS )    )annotationsN)Callable   )aio	shortuuid   )SentenceStream	TokenData
WordStreamc                   @  sn   e Zd Zddd ddZejd!ddZejd"ddZd"ddZd"ddZ	d"ddZ
d#ddZd$ddZdS )%BufferedTokenStreamF)retain_formattokenize_fncTokenizeCallablemin_token_lenintmin_ctx_lenr   boolreturnNonec                C  sD   t jt  | _|| _|| _|| _|| _t | _	g | _
d| _d| _d S )N )r   Chanr	   	_event_ch_tokenize_fnc_min_ctx_len_min_token_len_retain_formatr   _current_segment_id_buf_tokens_in_buf_out_buf)selfr   r   r   r    r!   f/var/www/html/livekit_bhavya/venv/lib/python3.10/site-packages/livekit/agents/tokenize/token_stream.py__init__   s   
zBufferedTokenStream.__init__textstrc                 C  s  |    |  j|7  _t| j| jk rd S 	 | | j}t|dkr$d S | jr.|  jd7  _|d}|}t|tr>|d }|  j|7  _t| j| j	kr\| j
t| j| jd d| _t|trl| j|d d  | _nt| j|d}| j|t| d   | _q)NTr    r   token
segment_idr   r   )_check_not_closedr   lenr   r   r   pop
isinstancetupler   r   send_nowaitr	   r   maxfindlstrip)r    r$   tokenstoktok_texttok_ir!   r!   r"   	push_text"   s2   


zBufferedTokenStream.push_textc                 C  s   |    | js
| jrL| | j}|r=| jr|  jd7  _t|d tr3|  jddd |D 7  _n
|  jd|7  _| jrL| jt	| j| j
d t | _
d| _d| _d S )Nr&   r   c                 S  s   g | ]}|d  qS )r   r!   ).0r4   r!   r!   r"   
<listcomp>P   s    z-BufferedTokenStream.flush.<locals>.<listcomp>r'   r   )r*   r   r   r   r-   r.   joinr   r/   r	   r   r   )r    r3   r!   r!   r"   flushE   s     
zBufferedTokenStream.flushc                 C  s   |    | j  d S N)r;   r   closer    r!   r!   r"   	end_input]   s   zBufferedTokenStream.end_inputc                   s   | j   d S r<   )r   r=   r>   r!   r!   r"   aclosea   s   zBufferedTokenStream.aclosec                 C  s,   | j jrt| }t|j d|j dd S )N.z
 is closed)r   closedtypeRuntimeError
__module____name__)r    clsr!   r!   r"   r*   d   s   z%BufferedTokenStream._check_not_closedc                 C  s   | S r<   r!   r>   r!   r!   r"   	__aiter__i   s   zBufferedTokenStream.__aiter__r	   c                   s   | j  I d H S r<   )r   	__anext__r>   r!   r!   r"   rI   l   s   zBufferedTokenStream.__anext__N)
r   r   r   r   r   r   r   r   r   r   )r$   r%   r   r   )r   r   )r   r   )r   r	   )rF   rE   __qualname__r#   typingno_type_checkr7   r;   r?   r@   r*   rH   rI   r!   r!   r!   r"   r      s    "



r   c                         e Zd Zd
 fdd	Z  ZS )BufferedSentenceStream	tokenizerr   r   r   r   r   r   c                     t  j|||d d S N)r   r   r   superr#   r    rO   r   r   	__class__r!   r"   r#   q   
   
zBufferedSentenceStream.__init__rO   r   r   r   r   r   r   r   rF   rE   rJ   r#   __classcell__r!   r!   rU   r"   rN   p       rN   c                      rM   )BufferedWordStreamrO   r   r   r   r   r   r   c                  rP   rQ   rR   rT   rU   r!   r"   r#      rW   zBufferedWordStream.__init__rX   rY   r!   r!   rU   r"   r\      r[   r\   )
__future__r   rK   collections.abcr   utilsr   r   rO   r   r	   r
   r%   listr.   r   r   r   rN   r\   r!   r!   r!   r"   <module>   s    $b