o
    :i                     @   sZ  d dl Z d dlmZ d dlmZmZ d dlmZ d dlm	Z	 		d&ded	e
d
ee	j dededdfddZ			d'dededee
 dB dededdfddZded	e
defddZ	d(dedee
 dB dee
 fddZdedee
 dee
 fddZdeded	e
ddfdd Zded	e
d!ee
e	jf ddfd"d#Z	d)deded	e
deddf
d$d%ZdS )*    N)Iterable)to_dictmodel_fields)
QdrantBase)models         @clientcollection_namepointsmax_attemptspausereturnc              
   C   s   d}||krJz| j ||dd W d S  tyE } z(td| d| d|  ||k r7td| d t| |d7 }W Y d }~nd }~ww ||kstd	| d
)N   T)r
   r   waitzException: z
, attempt /zNext attempt in z secondszFailed to upload points after z	 attempts)upload_points	Exceptionprinttimesleep)r	   r
   r   r   r   attemptse r   j/home/aiteam/pcaa-dev/dashboard-backend/venv/lib/python3.10/site-packages/qdrant_client/migrate/migrate.pyupload_with_retry	   s&   
r   Fd   source_clientdest_clientcollection_namesrecreate_on_collision
batch_sizec                    s   t  |}t fdd|D rtdt||}t|t| }|r,|s,td| |D ]}t || t ||| q.|D ]}t || t ||| q@dS )aE  
    Migrate collections from source client to destination client

    Args:
        source_client (QdrantBase): Source client
        dest_client (QdrantBase): Destination client
        collection_names (list[str], optional): List of collection names to migrate.
            If None - migrate all source client collections. Defaults to None.
        recreate_on_collision (bool, optional): If True - recreate collection if it exists, otherwise
            raise ValueError.
        batch_size (int, optional): Batch size for scrolling and uploading vectors. Defaults to 100.
    c                 3   s    | ]}t  |V  qd S N)_has_custom_shards.0r
   r   r   r   	<genexpr>7   s    

zmigrate.<locals>.<genexpr>z@Migration of collections with custom shards is not supported yetz*Collections already exist in dest_client: N)_select_source_collectionsany
ValueError_find_collisionsset_recreate_collection_migrate_collection)r   r   r   r    r!   
collisionsabsent_dest_collectionsr
   r   r&   r   migrate#   s    

r1   c                 C   s"   |  |}t|jjdd tjjkS )Nsharding_method)get_collectiongetattrconfigparamsr   ShardingMethodCUSTOM)r   r
   collection_infor   r   r   r#   K   s
   
r#   c                    s\   |   j}dd |D  |d ur*t fdd|D s(J dt|t   |S  }|S )Nc                 S   s   g | ]}|j qS r   namer%   
collectionr   r   r   
<listcomp>W       z._select_source_collections.<locals>.<listcomp>c                 3   s    | ]}| v V  qd S r"   r   r$   source_collection_namesr   r   r'   Z   s    
z-_select_source_collections.<locals>.<genexpr>z)Source client does not have collections: )get_collectionscollectionsallr,   )r   r   source_collectionsr   r@   r   r(   S   s   

r(   c                 C   s,   |   j}dd |D }|t|@ }t|S )Nc                 S   s   h | ]}|j qS r   r:   r<   r   r   r   	<setcomp>e   r?   z#_find_collisions.<locals>.<setcomp>)rB   rC   r,   list)r   r   dest_collectionsdest_collection_namesexisting_dest_collectionsr   r   r   r+   c   s   
r+   c                 C   s   |  |}|j}|j}||r|| d }|jd ur.tjdi dd t|j	 D }|j
||jj|jj|jj|jj|jj|jjtjdi t|jtjdi t|jtjdi t|j|j|d t||| d S )Nc                 S   s$   i | ]\}}|t tjv r||qS r   )r   r   StrictModeConfig)r%   kvr   r   r   
<dictcomp>x   s
    z(_recreate_collection.<locals>.<dictcomp>)vectors_configsparse_vectors_configshard_numberreplication_factorwrite_consistency_factoron_disk_payloadhnsw_configoptimizers_config
wal_configquantization_configstrict_mode_configr   )r3   r5   payload_schemacollection_existsdelete_collectionrY   r   rK   r   itemscreate_collectionr6   vectorssparse_vectorsrQ   rR   rS   rT   HnswConfigDiffrU   OptimizersConfigDiffoptimizer_configWalConfigDiffrW   rX   _recreate_payload_schema)r   r   r
   src_collection_info
src_configsrc_payload_schemarY   r   r   r   r-   j   s6   



r-   rZ   c                 C   s8   |  D ]\}}| j|||jd u r|jn|jd qd S )N)
field_namefield_schema)r]   create_payload_indexr6   	data_type)r   r
   rZ   ri   
field_infor   r   r   re      s   re   c                 C   s   | j |ddd\}}t|||d |dur+| j |||dd\}}t|||d |dus| |j}||j}||ksEJ d| d| dS )	aF  Migrate collection from source client to destination client

    Args:
        collection_name (str): Collection name
        source_client (QdrantBase): Source client
        dest_client (QdrantBase): Destination client
        batch_size (int, optional): Batch size for scrolling and uploading vectors. Defaults to 100.
       T)limitwith_vectors)r	   r
   r   N)offsetro   rp   zCMigration failed, vectors count are not equal: source vector count z, dest vector count )scrollr   count)r   r   r
   r!   recordsnext_offsetsource_client_vectors_countdest_client_vectors_countr   r   r   r.      s   

r.   )r   r   )NFr   r"   )r   )r   typingr   qdrant_client._pydantic_compatr   r   qdrant_client.client_baser   qdrant_client.httpr   strPointStructintfloatr   rG   boolr1   r#   r(   r+   r-   dictPayloadIndexInfore   r.   r   r   r   r   <module>   s    


(	


&
