
    Zi\                        d dl Z d dlZd dlmZ d dlZd dlZd dlmZmZm	Z	m
Z
mZ d dlmZ d dlmZ d dlmZ d dlZd dlZd dlmZmZ d dlmZmZ d dlmZ d d	lmZ d dlm c m!c mZ d dl"m c m!c m#Z# d d
lm$Z$ d dl%m&Z&m'Z'm(Z( d dl)m*Z*m+Z+m,Z,m-Z-m.Z. d dl/Z/d dl0m1Z2 d dl3m4Z4 d dl5Z6d dl7Z7eg ef         Z8de/j9        v r! eddddddd e7j:                              gn  eddddddd e7j:                              gZ; ej<        de;          dej=        deeddf         fd            Z> ej?         ej@        d ejA        d          dd          d          ZBejC        dejD        de
ejE        ejF        f         fd             ZG e ejH         eG            d!d" d# f$          %          d&ed'e	e
ejE        ejF        f                  ddfd(            ZId&eddfd)ZJd&ed*eKd+ejF        d,eddf
d-ZLd. ZM G d/ d0e&          ZNd1ZO G d2 d3e(          ZPd4ejQ        d&eddfd5ZRd4ejQ        d&eddfd6ZSd&eddfd7ZTd4ejQ        d&eddfd8ZUdS )9    N)
Connection)	GeneratorCallableListTuplecast)UUID)given)	ClientAPI	ServerAPI)SettingsSystem)VectorReader)LocalSegmentManager)hashing_embedding_function)EmbeddingStateMachineStatestraceEmbeddingStateMachineBase)run_state_machine_as_testruleprecondition
initializeMultipleResults)Client)DefaultEmbeddingFunctionCHROMA_RUST_BINDINGS_TEST_ONLYz!chromadb.api.rust.RustBindingsAPIz chromadb.db.impl.sqlite.SqliteDBz7chromadb.segment.impl.manager.local.LocalSegmentManagerT)chroma_api_implchroma_sysdb_implchroma_producer_implchroma_consumer_implchroma_segment_manager_implallow_resetis_persistentpersist_directoryzchromadb.api.segment.SegmentAPImodule)scopeparamsrequestreturnc              #      K   | j         V  d S N)param)r(   s    `/root/projects/butler/venv/lib64/python3.11/site-packages/chromadb/test/property/test_persist.pysettingsr.   D   s      
-    
   with_hnsw_paramswith_persistent_hnsw_paramsmax_hnsw_batch_sizemax_hnsw_sync_thresholdcoll)keydrawc                      | t          j        dt          j        d          dd                    } | t          j        t          j        |                              }||fS )NTr0   r1   )
strategiescollectionsstjust
recordsets)r8   
collection	recordsets      r-   !collection_and_recordset_strategyrA   V   so     !(* "$&	
 	
 	
 J Z*27:+>+>??@@Iy  r/      c                     | d         j         S Nr   namexs    r-   <lambda>rI   k   s    QqTY r/   c                     | d         j         S rD   rE   rG   s    r-   rI   rI   k   s    !A$) r/   )min_size	unique_by)#collection_and_recordset_strategiesr.   rM   c                    t          |           }|                                 t          j        |          }|                                 |D ]\  }}|                    |j        |j        |j                  } |j	        di | t          j        ||           t          j        ||           t          j        ||           t          j        ||           t          j        |||j                   |                                 ~~t          |           }|                                 t          j        |          }|D ]\  }}|                    |j        |j                  }t          j        ||           t          j        ||           t          j        ||           t          j        ||           t          j        |||j                   |                                 ~~d S )NrF   metadataembedding_function)rQ   rF   rQ    )r   startClientCreatorfrom_systemresetcreate_collectionrF   rP   rQ   add
invariantscountmetadatas_matchdocuments_match	ids_matchann_accuracystopget_collection)	r.   rM   system_1client_1collection_strategyrecordset_strategyr6   system_2client_2s	            r-   test_persistrh   g   s5    hHNN(22HNN 
-
 
 	))$)(12E * 
 
 	&&%&&&1222"4);<<<"4);<<<T#56662E	
 	
 	
 	
 	
 MMOOOhHNN(22H
 
-
 
 	&&$)2E ' 
 
 	1222"4);<<<"4);<<<T#56662E	
 	
 	
 	
 	
 MMOOOr/   c                    t          |           }|                                 t          j        |          }|                    dddd          }|                    t                    }|                    |j        t                    dt          ffd} |            }|                    ddgd	gd
gg            |            |k    sJ  |            }|                    dgdgg            |            |k    sJ  |            }|                    g dd	gd
gdgg            |            |k    sJ  |            }|                    dgdgg           |                    ddgd	gd
gg            |            |k    sJ  |            }|                    dgdgg           |                    ddgd	gd
gg            |            |k    sJ  |            }d S )Ntest   )hnsw:batch_sizehnsw:sync_threshold)rF   rP   r)   c                      t          j        d           	 t          j                                                                       S # t          $ r Y dS w xY w)Ng?)timesleepospathgetmtime_get_metadata_fileFileNotFoundError)segments   r-   get_index_last_modified_atz7test_sync_threshold.<locals>.get_index_last_modified_at   sY    
3	7##G$>$>$@$@AAA  	 	 	22	s   0A 
AA12      ?       @)ids
embeddings3      @)ry   rz   r   4      @5      @)r   rT   rU   rV   rX   instancer   get_segmentidr   floatrY   upsert)r.   systemclientr?   managerrx   last_modified_atrw   s          @r-   test_sync_thresholdr      s   HF
LLNNN&v..F))!ANN *  J oo122G!!*->>G       2133NNSzusenN=== &%''+;;;;;1133 NNu3%N111 &%''*:::::1133 ///usecU6KLLL &%''*:::::1133 NNu3%N1113*3%#@@@ &%''*:::::1133 NNu3%N111NNSzusenN=== &%''*:::::1133r/   collection_name
record_setconnc                    	 t          |           }|                                 t          j        |          }|                    |t          j                              }t          j        ||           t          j	        ||           t          j
        ||           t          j        ||           t          j        ||           |                                 d S # t          $ r}|                    |           |d }~ww xY w)NrR   )r   rT   rU   rV   ra   r:   "not_implemented_embedding_functionrZ   r[   r\   r]   r^   r_   r`   	Exceptionsend)r.   r   r   r   r   r   r6   es           r-   load_and_checkr      s    !!*622$$ )LNN % 
 
 	z***"4444"4444T:...j111   		!s   CC   
D*DDc                      	 t          j        d          } |                     dg           | S # t          $ r t          j        d          cY S w xY w)N
forkserverchromadbspawn)multiprocessingget_contextset_forkserver_preloadr   )ctxs    r-   get_multiprocessing_contextr     sh    
4 ),77 	""J<000
 4 4 4*7333334s   +. AAc                       e Zd ZdZdS )#PersistEmbeddingsStateMachineStatespersistN)__name__
__module____qualname__r   rS   r/   r-   r   r     s        GGGr/   r      c                        e Zd Zdedef fdZ ee          dej	        fd            Z e
d            e            dd
                        Zdedd	f fdZddZ xZS )PersistEmbeddingsStateMachiner   r.   c                     || _         || _        t          | _        | j                                          t                                          | j                    d S r+   )r   r.    MIN_STATE_CHANGES_BEFORE_PERSIST(min_state_changes_left_before_persistingrW   super__init__)selfr   r.   	__class__s      r-   r   z&PersistEmbeddingsStateMachine.__init__  sM     8X5%%%%%r/   r?   r?   c                 H   | j                                          | j                             |j        |j        |j                  | _        |j        | _        t          d           |                     t          j
                   t          j        g g g g           | _        d S )NrO   init)r}   	metadatas	documentsr~   )r   rW   rX   rF   rP   rQ   r?   r   on_state_changer   r   r:   StateMachineRecordSetrecord_set_state)r   r?   s     r-   r   z(PersistEmbeddingsStateMachine.initialize!  s    +77()< 8 
 

 #-"?f8CDDD * @bB2!
 !
 !
r/   c                 T    t          | j        d                   dk    o
| j        dk    S )Nr}   rB   r   )lenr   r   r   s    r-   rI   z&PersistEmbeddingsStateMachine.<lambda>2  s.    S.u566!; ?9Q> r/   r)   Nc                    |                      t          j                   | j        j        }t          j                    \  }}t                      }|                    t          | j
        || j        |f          }|                                 |                                 |                                r|                                }||                                 d S )N)targetargs)r   r   r   r?   rF   r   Piper   Processr   r.   r   rT   joinpollrecvclose)r   r   conn1conn2r   pr   s          r-   r   z%PersistEmbeddingsStateMachine.persist1  s     	@HIII/.&+--u)++KK!-$2GO  
 
 	
				::<< 	

AG						r/   	new_statec                     t                                          |           |t          j        k    rt          | _        d S | xj        dz  c_        d S )NrB   )r   r   r   r   r   r   )r   r   r   s     r-   r   z-PersistEmbeddingsStateMachine.on_state_changeH  sW    	***;CCC0 999 99Q>9999r/   c                 8    | j                                          d S r+   )r   rW   r   s    r-   teardownz&PersistEmbeddingsStateMachine.teardownQ  s    r/   )r)   N)r   r   r   r   r   r   r   collection_str:   
Collectionr   r   r   strr   r   __classcell__)r   s   @r-   r   r     s       &y &H & & & & & & Z=)))
Z%: 
 
 
 *)
 \	? 	?  
TVV   V	 
$? ? ? ? ? ? ? ?       r/   r   caplogc                     |                      t          j                   t          j                  t          fd           d S )Nc                  &    t                     S )Nr.   r   )r   )r   r.   s   r-   rI   z/test_persist_embeddings_state.<locals>.<lambda>[  s    -xOOO r/   )	set_levelloggingERRORr   r   r   )r   r.   r   s    `@r-   test_persist_embeddings_stater   U  sV     W]###_X&&FOOOOO    r/   c                 X   t          j        |          }t          ||          }|                    t	          j        dddddddd t          d          dt          j        i g dd	

  
                   |	                                 |
                                 |                                 |                                 |                                 |                    dgddggd gd d          \  }|	                                 |
                                 |                                 |                                 |                                 |                    ddgddgddggd d gd d          \  }}|	                                 |
                                 |                                 |                                 |                                 |                    |g           |	                                 |                                 d S )Nr   A00   rk   hnsw:construction_efhnsw:search_efhnsw:Mrm   rl   z$2d3eddc7-2314-45f4-a951-47a9a8e099d2   FT
rF   rP   rQ   r   	dimensiondtypeknown_metadata_keysknown_document_keywordshas_documentshas_embeddingsr   0g      ?g     ?r}   r~   r   r   r   ry   rz   g     P?g     ?g     ÿg     ?r}   )r   r   r   r   r:   r   r	   npfloat16r_   r[   fields_matchlog_size_below_maxno_duplicatesadd_embeddingsdelete_by_idsr   )r   r.   r   stateembedding_ids_0embedding_ids_1embedding_ids_2s          r-   test_delete_less_than_kr   _  s    _X&&F)8FKKKE	((+"%'(#$   $:;;* "$&!
 
 
    ( 
	KKMMM			--#XbdrWsVt  EI  DJ  Y]  :^  :^-  _  __		KKMMM			',';';PSUXzkx  {K  kL  O_  ao  Np  jq  AE  GK  @L  [_  H`  H`';  (a  (a$O_		KKMMM				_-...		NNr/   c                    t          j        |           }t          | |          }|                    t	          j        dddddddt                      t          d          dd i g dd	

  
                   |                    g dddgddgddggg dd d           |	                    dg           |                    dgddggd gd d           |
                                 d S )Nr   r   r   rk   )r   r   r   rl   rm   z$0851f751-2f11-4424-ab23-4ae97074887ar   FTr   r   )r   ry   rz   r   )NNNr   r   r   r   rB   )r   r   r   r   r:   r   r   r	   r   r   r   )r.   r   r   s      r-   test_delete_add_after_persistr     s`   _X&&F)8FKKKE	((+"%#$'(   899:;; "$&#
 
 
    , 
 #??q6Aq6Aq62+++
 
     
SE"""	5q6(	
 
     
r/   c                    t          j        |          }t          ||          }|                    t	          j        dddddddt          dt          j                  t          d	          dt          j        i g d
d
  
                   |
                                 |                                 |                                 |                                 |                                 t          t           t"                   |                    g dg dg dg dg dg dg dg dgd d d                    \  }}}}}}	}
|
                                 |                                 |                                 |                                 |                                 t'          d           |                    dgg dgd d d          }|
                                 |                                 |                                 |                                 |                                 t          t           t"                   |                    g dg dg dg dg dg d g d!g d"g d#g d$g d%g d&g d'g d(g d)g d*g d+g d,gd d d                    \  }}}}}}}}}}}}}}}}}|
                                 |                                 |                                 |                                 |                                 |                    d-d.gg d/g d0gd d gd d           |
                                 |                                 |                                 |                                 |                                 |                    g d1g d2g d3g d4g d5g d6g d7gd d d           |
                                 |                                 |                                 |                                 |                                 |                                 |
                                 |                                 |                                 |                                 |                                 |                    d8g d9gd d d           |
                                 |                                 |                                 |                                 |                                 |                    d:d;gg d<g d=gd d gd d           |
                                 |                                 |                                 |                                 |                                 |                    d>d?d@dA|dBdCdDdE||dFdGdH|dIdJgg dg dg dg dg d g d!g d"g d#g d$g d%g d&g d'g d(g d)g d*g d+g d,gd d d           |
                                 |                                 |                                 |                                 |                                 d S )KNr   JqzMs4pPm14cr   	      r   \   )dimr   z$45c5c816-0a90-4293-8d01-4325ff860040FTr   r   )Ne8r6r   YaoqFjA2cjHCvrz   )        r  r  )r{   r{   r{   )r|   r|   r|   )r   r   r   )r   r   r   )r   r   r   )      @r  r  r   r   z

	MVu393QTc)      @r  r  )CyF0Mk-q_Fwu2D2sQSFogDgPLkcfTSrwuQHQ6w4f51qWr2enLPQw8uKYs1Gwdzt5W8tpsnfJbV7zr   V1iFkoJXZw4uFc7vEEwrPYf)       @r  r  )      "@r  r  )      $@r  r  )      &@r  r  )      (@r  r  )      *@r  r  )      ,@r  r  )      .@r  r  )      0@r   r   )      1@r!  r!  )      2@r"  r"  )      3@r#  r#  )      4@r$  r$  )      5@r%  r%  )      6@r&  r&  )      7@r'  r'  )      8@r(  r(  r   df_RWhR0HelOcv)      9@r*  r*  )      :@r+  r+  )3R9_44u3BMZCXZDSUelx)      ;@r2  r2  )      <@r3  r3  )      =@r4  r4  )      >@r5  r5  )      ?@r6  r6  )      @@r7  r7  YlVm)     @@r9  r9  Rk1TPL)      A@r<  r<  )     A@r=  r=  r  r	  r
  r  r  r  r  r  r  r  r  r  r  )r   r   r   r   r:   r   r   r   float64r	   r_   r[   r   r   r   r   r   r   r   printr   r   )r   r.   r   r   r   r   r   embedding_ids_3embedding_ids_4embedding_ids_5embedding_ids_6_embedding_ids_12embedding_ids_17embedding_ids_18embedding_ids_22s                   r-   Mtest_batch_size_less_than_sync_with_duplicate_adds_results_in_skipped_seq_idsrI    sr    _X&&F)8FKKKE	((+"%'(#$   :b
SSS:;;* "$&!
 
 
    ( 
	KKMMM			 	GGG#OO#OO#OO#OO#OO#OO#OO "!  	 	
 	
	 	( 
	KKMMM				&MMM


=*??+	
 
   Q 
	KKMMM			( 	  ( $OO#OO&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&#& "!Q) ) 	 +	
 +	
.	 .	%													` 
	KKMMM				)*---/A/A/AB	
 
     
	KKMMM				???"""""""""""""""""" 
 
     
	KKMMM				MMOOO		KKMMM				---.	
 
     
	KKMMM				5>---/A/A/AB	
 
     
	KKMMM				 #/    #(  """""""""""""""""""""""""""""""""""""""""""""#& Q)
 )
  + + +X 
	KKMMM			NNr/   )Vr   r   multiprocessing.connectionr   multiprocessing.contextrp   typingr   r   r   r   r   uuidr	   
hypothesisr
   hypothesis.strategiesr:   r<   pytestr   chromadb.apir   r   chromadb.configr   r   chromadb.segmentr   #chromadb.segment.impl.manager.localr   !chromadb.test.property.strategiesrj   property!chromadb.test.property.invariantsrZ   r   &chromadb.test.property.test_embeddingsr   r   r   hypothesis.statefulr   r   r   r   r   rr   chromadb.api.clientr   rU   "chromadb.utils.embedding_functionsr   numpyr   tempfileCreatePersistAPIenvironmkdtempconfigurationsfixtureFixtureRequestr.   sharedr;   r=   r   	compositeDrawFnr   	RecordSetrA   listsrh   r   r   r   r   r   r   r   LogCaptureFixturer   r   r   rI  rS   r/   r-   <module>rj     s        1 1 1 1 1 1      9 9 9 9 9 9 9 9 9 9 9 9 9 9             " " " " " "   - - - - - - - - , , , , , , , , ) ) ) ) ) ) C C C C C C 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 H H H H H H         
              
			 7 7 7 7 7 7 G G G G G G     B	M*  (2:55 	?@!C!C(a.h.00		
 		
 		
  	=@!C!C(a.h.00		
 		
 		

 : h~666f+ 	(D$:N0O    76 	J$+BGDMM  "   	
 
 
 !
)!
: *"667! ! ! !  (0))++&&(;(;<) ) )  ==)-j#Z%99:*=
 
= = = =@84( 84t 84 84 84 84v $ 	
 
   64 4 4    *E    $%  9 9 9 9 9$= 9 9 9x$08	   .$.08.	. . . .d0H 0 0 0 0 0fQ$Q08Q	Q Q Q Q Q Qr/   