o
    i
                     @  s   d dl mZ d dlmZmZ d dlmZ d dlmZ ddl	m
Z
 g dZeG dd	 d	ZG d
d deZG dd deZG dd deZG dd deZdS )    )annotations)ABCabstractmethod)AsyncIterator)	dataclass   )aio)'!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~   ±u   —u   ‘u   ’u   “u   ”u   …c                   @  s&   e Zd ZU dZded< dZded< dS )	TokenData str
segment_idtokenN)__name__
__module____qualname__r-   __annotations__r.    r3   r3   c/var/www/html/livekit_bhavya/venv/lib/python3.10/site-packages/livekit/agents/tokenize/tokenizer.pyr*      s   
 r*   c                   @  s4   e Zd Zedddd	d
ZedddddZdS )SentenceTokenizerNlanguagetextr,   r7   
str | Nonereturn	list[str]c                C     d S Nr3   selfr8   r7   r3   r3   r4   tokenize      zSentenceTokenizer.tokenizeSentenceStreamc                C  r<   r=   r3   r?   r7   r3   r3   r4   stream   rA   zSentenceTokenizer.streamr8   r,   r7   r9   r:   r;   )r7   r9   r:   rB   )r/   r0   r1   r   r@   rD   r3   r3   r3   r4   r5      s
    r5   c                   @  s   e Zd ZdddZedddZedd	d
ZedddZedddZdddZ	d ddZ
dddZdddZed!ddZdS )"rB   r:   Nonec                 C     t jt  | _d S r=   r   Chanr*   	_event_chr?   r3   r3   r4   __init__!      zSentenceStream.__init__r8   r,   c                 C  r<   r=   r3   r?   r8   r3   r3   r4   	push_text$      zSentenceStream.push_textc                 C  r<   r=   r3   rK   r3   r3   r4   flush'   rP   zSentenceStream.flushc                 C  r<   r=   r3   rK   r3   r3   r4   	end_input*   rP   zSentenceStream.end_inputc                      d S r=   r3   rK   r3   r3   r4   aclose-      zSentenceStream.acloser*   c                      | j  I d H S r=   rJ   	__anext__rK   r3   r3   r4   rX   0      zSentenceStream.__anext__AsyncIterator[TokenData]c                 C     | S r=   r3   rK   r3   r3   r4   	__aiter__3   rP   zSentenceStream.__aiter__c                 C     | j   d S r=   rJ   closerK   r3   r3   r4   	_do_close6      zSentenceStream._do_closec                 C  ,   | j jrt| }t|j d|j dd S Nr   z
 is closedrJ   closedtypeRuntimeErrorr0   r/   r?   clsr3   r3   r4   _check_not_closed9      z SentenceStream._check_not_closedboolc                 C  s   | j jS r=   )rJ   re   rK   r3   r3   r4   re   >   s   zSentenceStream.closedNr:   rF   r8   r,   r:   rF   r:   r*   r:   rZ   )r:   rl   )r/   r0   r1   rL   r   rO   rQ   rR   rT   rX   r\   r`   rj   propertyre   r3   r3   r3   r4   rB       s     




rB   c                   @  s>   e Zd Zedddd	d
ZedddddZdddZdS )WordTokenizerNr6   r8   r,   r7   r9   r:   r;   c                C  r<   r=   r3   r>   r3   r3   r4   r@   D   rA   zWordTokenizer.tokenize
WordStreamc                C  r<   r=   r3   rC   r3   r3   r4   rD   H   rA   zWordTokenizer.streamwordsc                 C  s
   d |S )N )join)r?   rt   r3   r3   r4   format_wordsL   s   
zWordTokenizer.format_wordsrE   )r7   r9   r:   rs   )rt   r;   r:   r,   )r/   r0   r1   r   r@   rD   rw   r3   r3   r3   r4   rr   C   s    rr   c                   @  sv   e Zd ZdddZedddZedd	d
ZedddZedddZdddZ	dddZ
dddZdddZdS )rs   r:   rF   c                 C  rG   r=   rH   rK   r3   r3   r4   rL   Q   rM   zWordStream.__init__r8   r,   c                 C  r<   r=   r3   rN   r3   r3   r4   rO   T   rP   zWordStream.push_textc                 C  r<   r=   r3   rK   r3   r3   r4   rQ   W   rP   zWordStream.flushc                 C  r<   r=   r3   rK   r3   r3   r4   rR   Z   rP   zWordStream.end_inputc                   rS   r=   r3   rK   r3   r3   r4   rT   ]   rU   zWordStream.acloser*   c                   rV   r=   rW   rK   r3   r3   r4   rX   `   rY   zWordStream.__anext__rZ   c                 C  r[   r=   r3   rK   r3   r3   r4   r\   c   rP   zWordStream.__aiter__c                 C  r]   r=   r^   rK   r3   r3   r4   r`   f   ra   zWordStream._do_closec                 C  rb   rc   rd   rh   r3   r3   r4   rj   i   rk   zWordStream._check_not_closedNrm   rn   ro   rp   )r/   r0   r1   rL   r   rO   rQ   rR   rT   rX   r\   r`   rj   r3   r3   r3   r4   rs   P   s    



rs   N)
__future__r   abcr   r   collections.abcr   dataclassesr   utilsr   PUNCTUATIONSr*   r5   rB   rr   rs   r3   r3   r3   r4   <module>   s    
#