
    Zi^!              
       :   d dl Z d dlmZmZmZ d dlZd dlZd dlmZm	Z	m
Z
mZmZmZmZ d dlZd dlmZ d dlmc mc mZ d dlmZ d dlmZ d dlmZ d dlmZ d	eeef         fd
Zdeded	efdZ dedededed	df
dZ!dedededed	df
dZ"ded	dfdZ#ded	dfdZ$dS )    N)FutureThreadPoolExecutorwait)AnyDictListOptionalSetTuplecast)	ClientAPI)
SegmentAPI)	RecordSet)test_hnsw_config)Metadatareturnc                  ^    t          j        dd          } t          j        dd          }| |fS )N
   i'     )randomrandint)NDs     ]/root/projects/butler/venv/lib64/python3.11/site-packages/chromadb/test/test_multithreaded.pygenerate_data_shaper      s-    r5!!Ar3Aq6M    r   r   c                 &   d t          |           D             }d t          |           D             }d t          |           D             }t          j                            | |                                          }t          ||||          }|S )Nc                 ,    g | ]}t          |          S  )str.0is     r   
<listcomp>z'generate_record_set.<locals>.<listcomp>   s    
$
$
$a3q66
$
$
$r   c                     g | ]}| |iS r   r   r!   s     r   r$   z'generate_record_set.<locals>.<listcomp>   s    &E&E&Eq1{&E&E&Er   c                     g | ]}d | S )zdoc r   r!   s     r   r$   z'generate_record_set.<locals>.<listcomp>   s    ......r   ids
embeddings	metadatas	documents)rangenpr   randtolistr   )r   r   r(   r*   r+   r)   normalized_record_sets          r   generate_record_setr1      s    
$
$588
$
$
$C&E&EE!HH&E&E&EI..U1XX...I1%%,,..J (1	( ( ( ! r   clientnum_workersc           
         t          ||          }|d         }|d         }|d         }|d         }t          d| d| d| d           |                                  |                     d	t          
          }	t          |          5 }
g }d}|t          |          k     rt          j        d|dz            }t          |t          |          |z
            }|dz   }||z   dz   }|t          |||                   dk    rnx|

                    |	j        |||         |
|||         nd |
|||         nd |
|||         nd           }|                    |           ||z  }|t          |          k     d d d            n# 1 swxY w Y   t          |           |D ]}|                                }||t          j        |	|           t          j        |	|           t          j        |	|           t          j        |	           t          j        d t+          |          D             d          }d}t          j        |	|||           d S )Nr(   r)   r*   r+   Adding  records with  dimensions on  workerstestnamemetadatamax_workers   r   r   r'   c                     g | ]}|S r   r   r!   s     r   r$   z+_test_multithreaded_add.<locals>.<listcomp>[       "7"7"71"7"7"7r      	n_resultsquery_indices)r1   printresetcreate_collectionr   r   lenr   r   minsubmitaddappendr   	exception
invariantscount	ids_matchmetadatas_matchno_duplicatessampler,   ann_accuracy)r2   r   r   r3   records_setr(   r)   r*   r+   collexecutorfutures
total_sent
batch_sizeto_sendstartendfuturerO   rF   rE   s                        r   _test_multithreaded_addra   )   s    &a++K
e
C\*JK(IK(I	
LA
L
LQ
L
L{
L
L
LMMM LLNNN##:J#KKD		4	4	4 "%'
3s88##1733J*c#hh&;<<GNEw&*C%#js.C*D*D*I*I__c	N4>4J:eCi00PT2;2G)E#I..T2;2G)E#I..T %  F NN6"""'!J! 3s88##" " " " " " " " " " " " " " "* 	MMM  $$&&	 O ! T;'''{+++t[111T""" M"7"7eAhh"7"7"7<<MI#	     s   
C3F		FFc           	         |                                   |                     dt                    t          ||          }t	          t
          t                   |d                   t	          t
          t                   |d                   t	          t
          t                   |d                   |d         t                      t          j                    t          d| d| d	| d
           	 ddt          dt          t
          t                            ddffd}t          |          5 }g }d}|t!                    dz
  k     rt#          j        dd          }	|	dk    rt#          j        d|dz            }
t'          |
t!                    |z
            }|dz   }||z   dz   }|                    ||	||                   }|                    |           ||z  }n1|	dk    r+|                    ||	          }|                    |           |t!                    dz
  k     ddd           n# 1 swxY w Y   t-          |           |D ]}|                                }||t1          | t2                    r/|                                 j        du rt9          j        |           t9          j        |           t9          j        |           t9          j         |           t9          j!                   t#          j"        d tG          |          D             d          }d}t9          j$        |||           dS )zeTest that will use multiple threads to interleave operations on the db and verify they work correctlyr9   r:   r(   r)   r*   r+   r5   r6   r7   r8   N	operationids_to_modifyr   c           	         | dk    r|J fd|D             }t          |          dk    rdS 	                    |fd|D             ndfd|D             nd

fd|D             nd           5                      |           ddd           dS # 1 swxY w Y   dS | dk    rg }d	}5  t                                                    }ddd           n# 1 swxY w Y   fd
|D             }t          |          dk    st          |          |k     rdS t          j        |t          dt          |                              }fd|D             }	                    ||           dS dS )z,Perform a random operation on the collectionr   Nc                 :    g | ]}                     |          S r   indexr"   idr(   s     r   r$   zJ_test_interleaved_add_query.<locals>.perform_operation.<locals>.<listcomp>~   s#     G G G22 G G Gr   c                      g | ]
}|         S r   r   r"   r#   r)   s     r   r$   zJ_test_interleaved_add_query.<locals>.perform_operation.<locals>.<listcomp>   s    EEEaJqMEEEr   c                      g | ]
}|         S r   r   )r"   r#   r*   s     r   r$   zJ_test_interleaved_add_query.<locals>.perform_operation.<locals>.<listcomp>       CCCA9Q<CCCr   c                      g | ]
}|         S r   r   )r"   r#   r+   s     r   r$   zJ_test_interleaved_add_query.<locals>.perform_operation.<locals>.<listcomp>   rn   r   r'   r@   rC   c                 :    g | ]}                     |          S r   rg   ri   s     r   r$   zJ_test_interleaved_add_query.<locals>.perform_operation.<locals>.<listcomp>   s#    &S&S&Ssyy}}&S&S&Sr   r   c                      g | ]
}|         S r   r   rl   s     r   r$   zJ_test_interleaved_add_query.<locals>.perform_operation.<locals>.<listcomp>   s    BBBqZ]BBBr   )rE   )	rJ   rM   updatelistcopyr   rU   rK   query)rc   rd   indices_to_modifycurrently_added_idsrE   currently_added_indicesrF   query_vectors	added_idsrX   r+   r)   r(   lockr*   s           r   perform_operationz6_test_interleaved_add_query.<locals>.perform_operationx   s    >> ,,, G G G G G G G$%%**HH!) FEEE3DEEEE( DCCC1BCCCC( DCCC1BCCCC      0 0  ///0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0!^^"$I = =&*9>>+;+;&<&<#= = = = = = = = = = = = = = =&S&S&S&S?R&S&S&S#'((A--.//);;
 #M'B34455 M CBBBMBBBMJJ#      + ^s$   :BB!$B!6"C$$C(+C(r=   r?   r@   r      r   Tc                     g | ]}|S r   r   r!   s     r   r$   z/_test_interleaved_add_query.<locals>.<listcomp>   rB   r   rC   rD   )N)%rH   rI   r   r1   r   r   r    floatr   set	threadingLockrG   intr	   r   rJ   r   r   rK   rL   rN   r   rO   
isinstancer   get_settingsis_persistentrP    fd_not_exceeding_threadpool_sizerQ   rR   rS   rT   rU   r,   rV   )r2   r   r   r3   rW   r|   rY   rZ   r[   rc   r\   r]   r^   r_   r`   rO   rF   rE   rz   rX   r+   r)   r(   r{   r*   s                     @@@@@@@r   _test_interleaved_add_queryr   e   s   
 LLNNN##:J#KKD%a++K
tCy+e,
-
-Cd5k;|#<==JT(^[%=>>IK(I%%I>D	
LA
L
LQ
L
L{
L
L
LMMM >B0 00'/S	':0	0 0 0 0 0 0 0 0 0 0 0 0d 
	4	4	4 '%'
3s88a<''q!,,IA~~#^AqBw77
j#c((Z*?@@"Q 7*Q.!):Is5QT9~VVv&&&g%

a!%  v&&&! 3s88a<''' ' ' ' ' ' ' ' ' ' ' ' ' ' '* 	MMM  $$&&	 O ! 	6:&&A+1+>+>+@+@+NRV+V+V3K@@@T;'''{+++t[111T""" M"7"7eAhh"7"7"7<<MI#	     s    C<IIIc                     t          d          D ]N}t          j        dt          j                    dz            }t                      \  }}t          | |||           Od S N   r}   )r,   r   r   multiprocessing	cpu_countr   ra   r2   r#   r3   r   r   s        r   test_multithreaded_addr      sf    1XX ; ;nQ(A(C(Ca(GHH"$$11k::::; ;r   c                     t          d          D ]N}t          j        dt          j                    dz            }t                      \  }}t          | |||           Od S r   )r,   r   r   r   r   r   r   r   s        r   test_interleaved_add_queryr      sf    1XX ? ?nQ(A(C(Ca(GHH"$$1#FAq+>>>>? ?r   )%r   concurrent.futuresr   r   r   r   r   typingr   r   r   r	   r
   r   r   numpyr-   chromadb.apir   !chromadb.test.property.invariantsr9   propertyrP   chromadb.api.segmentr   !chromadb.test.property.strategiesr   r   chromadb.typesr   r   r   r1   ra   r   r   r   r   r   r   <module>r      s!       ? ? ? ? ? ? ? ? ? ?      > > > > > > > > > > > > > > > > > >     " " " " " " 6 6 6 6 6 6 6 6 6 6 6 6 + + + + + + 7 7 7 7 7 7 > > > > > > # # # # # #U38_    !3 !3 !9 ! ! ! !&999"%9479	9 9 9 9xrrr"%r47r	r r r rj;9 ; ; ; ; ;?y ?T ? ? ? ? ? ?r   