
    Zi^                        d dl Z d dlZd dlZd dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlZd dlmZ d dlmZmZ d dlmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z% d dl&m'Z' d dl(Z)d dl*mZ+ d dl,m-Z-m.Z. d dl/m0Z0 d dl1m2Z2 d dl3m4Z4 d dl5m6Z6m7Z7 d dl8m9Z9 d dl:m;Z; d dl<m=Z=m>Z>m?Z?m@Z@  e"d          ZAde eAe!eA         f         de!eA         fdZBdedefdZCdee-jD                 dee-jD                 ddfdZEd e0deddfd!ZFd e0d"ed#e e'd$         e'd%         e'd&         f         ddfd'ZGd e0deddfd(ZHd e0deddfd)ZId e0deddfd*ZJd e0deddfd+ZKd e0ddfd,ZLe9jM        fd-e-jN        d.e-jN        d/ee+jO        e+jO        gePf         dee!e!eQ                  e!e!eP                  f         fd0ZRd1eQddfd2ZSd e0fd3ZT	 	 	 	 	 	 dKd e0ded7eQd8ePd9ee-jU                 d:ee!eQ                  d;ee-jN                 d<eVddfd=ZWd>e-jX        d7eQddfd?ZYd@edeQfdAZZdBedCe!e0         dDeVddfdEZ[dBedCe!e0         de%eeQf         fdFZ\dBedCe!e0         dDeVddfdGZ]edHe.dIe^fdJ            Z_dS )L    N)UUID)contextmanager)
SegmentAPI)SysDB)create_topic_name)System)get_sql)SqliteDB)sleep)SegmentType)NormalizedRecordSet	RecordSet)	CallableOptionalTupleUnionListTypeVarcastAnyDict)Literal)types	ClientAPI)
Collection)note)InvalidArgument)Table	functions)distance_functions)Search)KnnSelectLimitKeyTvaluereturnc                 X    | t          d          t          | t                    r| S | gS )z*Wrap a value in a list if it is not a listNzvalue cannot be None)r   
isinstancer   )r'   s    ^/root/projects/butler/venv/lib64/python3.11/site-packages/chromadb/test/property/invariants.pywrapr,   $   s4    }4555	E4	 	  w    
record_setc                 X   | d         d}n?t          | d         t                    r| d         J t          | d                   dk    rt          d | d         D                       r!t	          t
          j        | d                   }nt          d | d         D                       r!t	          t
          j        | d                   }nt          d | d         D                       r"t	          t
          j        | d         g          }n?t          d          t	          t
          j        | d                   }nt          d          t          t          | d	                   | d
         t          | d
                   nd| d         t          | d                   nd|          S )z9Ensure that an embedding set has lists for all its values
embeddingsNr   c              3   @   K   | ]}t          |t                    V  d S N)r*   list.0	embeddings     r+   	<genexpr>zwrap_all.<locals>.<genexpr>7   s=        09
9d++     r-   c              3   J   K   | ]}t          |t          j                  V  d S r2   )r*   npndarrayr4   s     r+   r7   zwrap_all.<locals>.<genexpr>;   sB         9bj11     r-   c              3   z   K   | ]6}t          |t          t          t          j        t          j        f          V  7d S r2   )r*   intfloatr9   integerfloatingr5   es     r+   r7   zwrap_all.<locals>.<genexpr>A   sM         q3rz2;"GHH     r-   z-an embedding must be a list of floats or intszVembeddings must be a list of lists, a list of numpy arrays, a list of numbers, or Noneids	documents	metadatas)rB   rC   rD   r0   )
r*   r3   lenallr   r   
Embeddingsr   r   r,   )r.   embedding_lists     r+   wrap_allrI   .   s    ,'	J|,d	3	3 
,'333z,'((1,,  =G=U      "&e&6
<8P!Q!Q  !+L!9      "&e&6
<8P!Q!Q  '5      &*%*:Z=U<V%W%WNN)G   "%"2J|4LMMNNd
 
 	
 E"##k". z+.///k". z+.///!	 	 	 	r-   expectedgotc           	         | || |J | |t          |           t          |          k    sJ |                                 D ]\  }}||v sJ t          | |         t                    rft          ||         t                    rKt	          t          t          | |                   t          t          ||                   z
            dk     sJ | |         ||         k    sJ d S d S d S )Nư>)rE   itemsr*   r=   absr   )rJ   rK   keyvals       r+   check_metadatarR   ]   s    8}}C(((( (( 	1 	1HC#::::(3-// 1Js3x4O4O 14x}55UCH8M8MMNNQUUUUUU}C00000 	1 	1r-   
collectionc           	          |                                  }t          |          }|t          |d                   k    r%t          d|dt          |d                              |t          |d                   k    sJ dS )z?The given collection count is equal to the number of embeddingsrB   zcount mismatch:z=!N)countrI   rE   print)rS   r.   rU   normalized_record_sets       r+   rU   rU   m   s    E$Z00)%01111c2G2N.O.OPPPC-e455555555r-   rW   
field_namerC   rD   r0   c                   	 |                      |d         |g          	d t          |d                   D             	|         }t          |d                   dk    rA|dk    r1t          t          j        t                   |          j        dk    sJ n|g k    sJ dS |J t          t          |          	fd          }d	 |D             }||         }|dgt          |d                   z  }|dk    r=t          j
        t          j        |          t          j        |                    sJ dS t          |          t          |          k    sJ t          ||          D ]c\  }}t          |t                    rAt          t          t           j        |          t          t           j        |                     [||k    sJ ddS )
zq
    The actual embedding field is equal to the expected field
    field_name: one of [documents, metadatas]
    rB   rB   includec                     i | ]\  }}||	S  r]   r5   iids      r+   
<dictcomp>z"_field_matches.<locals>.<dictcomp>       XXXuq"RXXXr-   r   r0   Nc                 8    d         | d                           S )NrB   r   r]   )index_and_field_valueembedding_id_to_indexresults    r+   <lambda>z _field_matches.<locals>.<lambda>   s    *?5M/23+
 r-   rP   c                     g | ]\  }}|S r]   r]   )r5   _field_values      r+   
<listcomp>z"_field_matches.<locals>.<listcomp>   s    CCCNA{KCCCr-   )get	enumeraterE   r   nptNDArrayr   sizesortedr9   allclosearrayzipr*   dictrR   r   Metadata)
rS   rW   rX   actual_fieldsorted_fieldfield_valuesexpected_fieldrk   re   rf   s
           @@r+   _field_matchesr|   v   s     ^^ 5e <zl^SSF YX	:OPU:V0W0WXXX*%L
 '((A--%%C(,77<AAAAA2%%%% ###,
 
 
 
 
  L DClCCCL*:6N #&;E&B"C"CC\!!{28L1128N3K3KLLLLLLL<  C$7$77777+.|^+L+L 	5 	5'K.$// 55588   
 #n44444	5 	5r-   c                     t          |          }|                     |d         g           d         }d t          |d                   D             t          |fd          }||d         k    sJ dS )z5The actual embedding ids is equal to the expected idsrB   rZ   c                     i | ]\  }}||	S r]   r]   r^   s      r+   ra   zids_match.<locals>.<dictcomp>   rb   r-   c                     |          S r2   r]   )r`   re   s    r+   rg   zids_match.<locals>.<lambda>   s    3H3L r-   rh   N)rI   rm   rn   rr   )rS   r.   rW   
actual_idsre   s       @r+   	ids_matchr      s    $Z00$9%$@"MMeTJ YX	:OPU:V0W0WXXX
(L(L(L(LMMMJ.u5555555r-   c                 F    t          |          }t          | |d           dS )z?The actual embedding metadata is equal to the expected metadatarD   NrI   r|   rS   r.   rW   s      r+   metadatas_matchr      )    $Z00:4kBBBBBr-   c                 F    t          |          }t          | |d           dS )AThe actual embedding documents is equal to the expected documentsrC   Nr   r   s      r+   documents_matchr      r   r-   c                 F    t          |          }t          | |d           dS )r   r0   Nr   r   s      r+   embeddings_matchr      s)    $Z00:4lCCCCCr-   c                     |                                  d         }t          |          t          t          |                    k    sJ d S )NrB   )rm   rE   set)rS   rB   s     r+   no_duplicatesr      s?    
..

5
!Cs88s3s88}}$$$$$$r-   querytargetsdistance_fnc                 .   t          j        | t           j                  }t          j        |t           j                  t          j        fdd|          }t          j        |                                          |                                fS )zGReturn the ordered indices and distances from each query to each target)dtypec                 2    t          j        d|           S )N   )r9   apply_along_axis)r   r   
np_targetss    r+   rg   z"_exact_distances.<locals>.<lambda>   s    b)+q*eLL r-   r   )r9   rt   float32r   argsorttolist)r   r   r   np_query	distancesr   s     `  @r+   _exact_distancesr      s     xRZ000H'444J #LLLLL	 I :i  ''))9+;+;+=+===r-   threadpool_sizec                    t          j                    }|                                }d}d}t          d |D                       dz
  | k    rg||k     rat	          j                     |                                }|dz  }t          d           t          d |D                       dz
  | k    r||k     at          d |D                       dz
  | k    sJ dS )zs
    Checks that the open file descriptors are not exceeding the threadpool size
    works only for SegmentAPI
       r   c                 .    g | ]}d |j         v |j         S sqlite3pathr5   ps     r+   rl   z4fd_not_exceeding_threadpool_size.<locals>.<listcomp>   &    ???9+>+>QV+>+>+>r-   r   c                 .    g | ]}d |j         v |j         S r   r   r   s     r+   rl   z4fd_not_exceeding_threadpool_size.<locals>.<listcomp>   r   r-   N)psutilProcess
open_filesrE   gccollectr   )r   current_processr   max_retriesretry_counts        r+    fd_not_exceeding_threadpool_sizer      s    
 n&&O ++--JKK 	??Z???@@1DVV+%%

$//11
qa 	??Z???@@1DVV+%% 	??Z???@@1DWWWWWWr-   c                 X   d }d| j         v r| j         d         }| j        j        |S d| j        j        v rs| j        j                            d          Td| j        j                            d          v r3| j        j                            d                              d          }nd| j        j        v rt| j        j                            d          Ud| j        j                            d          v r4|2| j        j                            d                              d          }|S )Nz
hnsw:spacespannspacehnsw)metadata_modelconfiguration_jsonrm   )rS   r   s     r+   	get_spacer      s#    Ez***#L1+3:$777044W==Iz(;??HHHH!488AAEEgNN*#666044V<<Hz(;??GGGG=%8<<VDDHHQQELr-   r   ffffff?F	n_results
min_recallembedding_functionquery_indicesquery_embeddings
use_searchc           
      &
  $% t          |          }t          |d                   dk    rdS |d         $$duot          $          dk    }	|	s<|J |d         J t          |d         t                    sJ  ||d                   $t	          |           }
|
t
          j        }n8|
dk    rt
          j        }n%|
dk    rt
          j        }n|
dk    rt
          j        }d	}| j	        J $J t          d
 $D                       sJ t          $d                   }|t          j        dt          t          j        |                              z  }||$n$fd|D             }|d         %|%%fd|D             %t          |$|          \  }}|rg }|D ]}t          |t           j                  r|                                }n|}t'          t)          |          t+          |                                                    }|                    |           |                     |          }t3          t4          j        |d         |d         |d         |d         |d         d          }n#|                     |	r|nd|	s%nd|g d          }t;          ||           |d         J |d         J |d         J |d         J |d         J d t=          |d                   D             }d}t=          t?          ||                    D ]j\  }\  }}t!          j         |d                   |d|                  }|t          tC          |          tC          |d         |                   z
            z  }t=          |d         |                   D ]\  }}||v}||         }t!          j"        ||         |d         |         |         |          } |r	| r|dz  }nK| sJ t!          j"        $|         |d         |         |                   sJ |d         &|d         |         |d         |         |         k    sJ |d         .tG          |d         |         |d         |         |                    lt          |d                   }!|!|z
  |!z  }"	 tI          d|" d| d|! d|            n# tJ          $ r Y nw xY w|"|k    sJ |d         D ]+}#t!          j"        t!          j&        |#          |#          sJ ,dS ) zBValidate that the API performs nearest_neighbor searches correctlyrB   r   Nr0   rC   cosineipl2rM   c              3   X   K   | ]%}t          |t          t          j        f          V  &d S r2   )r*   r3   r9   r:   r@   s     r+   r7   zann_accuracy.<locals>.<genexpr>>  s3      EEQz!dBJ/00EEEEEEr-   
   c                      g | ]
}|         S r]   r]   )r5   r_   r0   s     r+   rl   z ann_accuracy.<locals>.<listcomp>G  s    777A*Q-777r-   c                      g | ]
}|         S r]   r]   )r5   r_   query_documentss     r+   rl   z ann_accuracy.<locals>.<listcomp>K  s    EEE!?1-EEEr-   )r   )r   )limit)rankr   )searchesscoresrD   )rB   r   r0   rC   rD   )r0   rC   rD   r   )r   query_textsr   r[   r   c                     i | ]\  }}||	S r]   r]   r^   s      r+   ra   z ann_accuracy.<locals>.<dictcomp>  s    NNNUQ2qNNNr-   )atolr   z
# recall: z
, missing z out of z, accuracy threshold )'rI   rE   r*   r3   r   r    r   r   r   r   rF   mathpowr<   log10r   r9   r:   r   r!   r"   r$   
select_allappendsearchr   r   QueryResultr    _query_results_are_correct_shapern   ru   rt   r   rs   rR   r   r   sort)&rS   r.   r   r   r   r   r   r   rW   have_embeddingsr   distance_functionaccuracy_thresholddimindicesr   search_requestsquery_embeddingquery_embedding_listr   search_resultsquery_resultsid_to_indexmissingr_   	indices_idistances_iexpected_idsjr`   unexpected_idindexcorrect_distancerq   recalldistance_resultr0   r   s&                                       @@r+   ann_accuracyr     s    %Z00
 '((A---B<-PJ ,DZ11DO L!---$[1===/<dCCCCC''(=k(JKK
j!!E}.1	(		.5	$.1	$.1***!!!
 EE*EEEEEEEE
jm

C+dhr3tz#;O;O.P.PP  $ J7777777 	
 ,K8O _%@EEEE}EEE)*2C  GY  '
/ 
	+ 
	+O/2:66 7'6'='='?'?$$'6$3444),,,   jll  ""6**** $**$ + 
 

 %e,+ -\:+K8+K8 
 
 #((1@J--d/>HDIII	 ) 
 
 %]I>>> +++%111&222%111%111 ONi0Ee0L&M&MNNNKG'0Wi1H1H'I'I $ $##I{x 5e <==i

>ST3s<((3}U/CA/F+G+GGHHH }U3A677 	 	EArl2MOE!{E"k*1-a0'     
  ( $ qLGG'''';z%0-2Ma2PQR2STTTTT$[1=)+6u=$[1!4Q78 8 8 8 %[1=)+6u=!+.q1!4  7	@ $U+,,DWn$Fkkk7kkDkkWikk	
 	
 	
 	
     Z )5 F F{27?33_EEEEEEF Fs   )S 
SSr   c                 l    dD ]/}| |         J t          fd| |         D                       sJ 0d S )N)r   r0   rC   rD   c              3   >   K   | ]}t          |          k    V  d S r2   )rE   )r5   rf   r   s     r+   r7   z3_query_results_are_correct_shape.<locals>.<genexpr>  s?       
 
)/CKK9$
 
 
 
 
 
r-   )rF   )r   r   result_types    ` r+   r   r     s|     M 
 
[)555 
 
 
 
3@3M
 
 
 
 
 	
 	
 	
 	

 
r-   sqlitec                    t          d          }|                                                     |          }|                                 5 }t	          |                    t          j        |j                            | 	                                          \  }}|
                    ||          }t          t          |                                d                   cd d d            S # 1 swxY w Y   d S )Nembeddings_queuer   )r   querybuilderfrom_txr	   selectr   Countseq_idparameter_formatexecuter   r<   fetchone)r   tqcursqlparamsrf   s          r+   _total_embedding_queue_log_sizer    s    !!A##A&&A	 /HHY_QX..//1H1H1J1J
 
V S&))C**1-../ / / / / / / / / / / / / / / / / /s   BC**C.1C.systemcollectionshas_collection_mutatedc                    |                      t                    }| j        j        s| j        j        dk    rd S |rxt          |          dk    sJ t          d |D                       }t          d |D                       }| j        j        dk    r|n||z   }t          |          dz
  |k    sJ d S t          |          dk    sJ d S )N!chromadb.api.rust.RustBindingsAPIr   c              3   ^   K   | ](}|j         |j                             dd          ndV  )d S )Nhnsw:sync_threshold  r   rm   r5   rS   s     r+   r7   z%log_size_below_max.<locals>.<genexpr>  s]       !
 !
  ". ##$94@@@!
 !
 !
 !
 !
 !
r-   c              3   ^   K   | ](}|j         |j                             dd          ndV  )d S Nzhnsw:batch_sized   r  r  s     r+   r7   z%log_size_below_max.<locals>.<genexpr>  s]       
 
  ". ##$5s;;;
 
 
 
 
 
r-   r   )instancer
   settingsis_persistentchroma_api_implr  sum)r  r  r  r   sync_threshold_sumbatch_size_sumr   s          r+   log_size_below_maxr    s'    __X&&F O)O+/RRR <.v66!;;;; ! !
 !
 *	!
 !
 !
 
 
  
 
 *	
 
 
 
 
 .2UUU #n4 	 /v66:eCCCCCC.v66!;;;;;;r-   c           	         |                      t                    }t          d          }|                                                    |          }| j                            d          | j                            d          fd|D             }|                                5 }t          |	                    |j
        t          j        |j                                                d          |                                          \  }}|                    ||          }	i }
|	                                D ]}|d         |
||d                  j        <   |
cd d d            S # 1 swxY w Y   d S )Nr   	tenant_idtopic_namespacec                 >    i | ]}t          |j                  |S r]   )r   r`   )r5   rS   _tenant_topic_namespaces     r+   ra   zB_total_embedding_queue_log_size_per_collection.<locals>.<dictcomp>  s:        	'#3Z]CCZ  r-   topicr   r   )r  r
   r   r   r   r  requirer   r	   r   r!  r   r   r   groupbyr   r   fetchallr`   )r  r  r   r   r  topic_mappingsr  r  r  rf   outresr  r   s               @@r+   ._total_embedding_queue_log_size_per_collectionr(    s    __X&&F !!A##A&&Ao%%k22G../@AA    %  N 
 	HHQWioah7788@@II##%%
 
V S&))??$$ 	4 	4C-0VCs1v&)**	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   *B5E,,E03E0c                    | j         j        dk    rd S |                     t                    }|rt	          |          dk    sJ d |D             fd|D             t          | |          t                                                    t                                                    k    sJ t          fd|D                       sJ d S t	          |          dk    sJ d S )Nr
  r   c                 b    i | ],}|j         |j        |j                            dd          nd-S r  )r`   r   rm   r  s     r+   ra   z;log_size_for_collections_match_expected.<locals>.<dictcomp>$  sT     
 
 
  M". &.223DcJJJ
 
 
r-   c                 f    i | ]-}|j         |                                |j                  z  d z   .S )r   )r`   rU   )r5   rS   r  s     r+   ra   z;log_size_for_collections_match_expected.<locals>.<dictcomp>*  sK     
 
 
 M:++--z}0MMPQQ
 
 
r-   c              3   P   K   | ] }|j                  |j                  k    V  !d S r2   )r`   )r5   rS   actual_sizesexpected_sizess     r+   r7   z:log_size_for_collections_match_expected.<locals>.<genexpr>3  sJ       
 
 '>*-+HH
 
 
 
 
 
r-   r   )	r  r  r  r
   r  r(  r   keysrF   )r  r  r  r   r-  r  r.  s       @@@r+   'log_size_for_collections_match_expectedr0    s[    &*MMM__X&&F <.v66!;;;;
 
 *	
 
 

 
 
 
)
 
 

 FK
 
 <$$&&''3~/B/B/D/D+E+EEEEE 
 
 
 
 
)
 
 
 
 
 	
 	
 	
 	
 	
 /v66!;;;;;;r-   clientcollection_namec           
   #     K   d |                                  D             }||v sJ |                     |          }g }t          | j        t                    r,| j        j        }|                    |j                  }i }d}|D ]}d||d         <   |d         t          j	        j
        k    r|j        d|j        v r|j        d         nd}	|                                |	k    rkd}t          j                            t          j                            |                                 j        t'          |d                                       sJ |r|t          j	        j
                 sJ |t          j        j
                 sJ d V  d	 |                                  D             }||vsJ t+          |          d
k    r| j        j        }|                    |j                  }
t+          |
          d
k    sJ |D ]}|d         t          j	        j
        k    rit          j                            t          j                            |                                 j        t'          |d                                       rJ d S d S )Nc                     g | ]	}|j         
S r]   namer5   cs     r+   rl   z&collection_deleted.<locals>.<listcomp>?      BBB1BBBr-   )rS   FTtyper  r  r`   c                     g | ]	}|j         
S r]   r5  r7  s     r+   rl   z&collection_deleted.<locals>.<listcomp>a  r9  r-   r   )list_collectionsget_collectionr*   _serverr   _sysdbget_segmentsr`   r   HNSW_LOCAL_PERSISTEDr'   r   rU   osr   existsjoinget_settingspersist_directorystrSQLITErE   )r1  r2  collection_namesrS   segmentssysdbsegment_typesshould_have_hnswsegmentsync_thresholdsegments_afters              r+   collection_deletedrQ  <  s      CB(?(?(A(ABBB.....&&77JH&.*-- 7~,%%%??  	 	G-1M'&/*v+"B"HHH "*6-1DDD '(=>> 	  $$&&77'+$7>>"//11CSQUEWEW     
  	I !A!GHHHH[/56666	EEE CB(?(?(A(ABBB"22222
8}}q~,++z}+EE>""a'''' 	 	Gv+"B"HHH7>>GLL++--?WT]ASAS      	 	r-   )r   r   NNNF)`r   r   os.pathrB  uuidr   
contextlibr   chromadb.api.segmentr   chromadb.db.systemr   chromadb.ingest.impl.utilsr   chromadb.configr   chromadb.db.baser	   chromadb.db.impl.sqliter
   timer   r   chromadb.segmentr   !chromadb.test.property.strategiesr   r   typingr   r   r   r   r   r   r   r   r   typing_extensionsr   numpyr9   numpy.typingro   chromadb.apir   r   chromadb.api.models.Collectionr   
hypothesisr   hypothesis.errorsr   pypikar   r   chromadb.utilsr    "chromadb.execution.expression.planr!   &chromadb.execution.expression.operatorr"   r#   r$   r%   r&   r,   rI   rw   rR   rU   r|   r   r   r   r   r   r   rG   	ArrayLiker=   r<   r   r   r   EmbeddingFunctionboolr   r   r   r  r  r(  r0  rG  rQ  r]   r-   r+   <module>rm     s   				         % % % % % % + + + + + + $ $ $ $ $ $ 8 8 8 8 8 8 " " " " " " $ $ $ $ $ $ , , , , , ,        ( ( ( ( ( ( L L L L L L L L S S S S S S S S S S S S S S S S S S S S S S % % % % % %           ) ) ) ) ) ) ) ) 5 5 5 5 5 5       - - - - - - # # # # # # # # - - - - - - 5 5 5 5 5 5 J J J J J J J J J J J JGCLLaaj! d1g    , ,': , , , ,^1u~&1-5en-E1	1 1 1 1 6j 6i 6D 6 6 6 64545.45 gk2GL4II45 
45 45 45 45n6* 6) 6 6 6 6 6C
 C	 Cd C C C CC
 C	 Cd C C C CD D Dt D D D D%j %T % % % % 	> >>> 	&-> 4S	?De--.> > > >*c d    .*    8 <@)-37fF fFfFfF fF 	fF
 !!89fF DI&fF u/0fF fF 
fF fF fF fFR
$
14
	
 
 
 
	/H 	/ 	/ 	/ 	/ 	/(<(<!%j!1(<KO(<	(< (< (< (<Vj! 
$)_   2"<"<!%j!1"<KO"<	"< "< "< "<J 0y 03 0 0 0 0 0 0r-   