
    ZiP                     r   d dl mZ d dlmZmZmZmZ d dlmZ d dl	m	Z	 d dl
Z
d dlmZmZmZmZ d dlmZmZmZmZmZmZ d dlmc mZ d dlmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9 d d	l:m;Z;m<Z< d dl=Z=d d
l>m?Z? d dl@mAZAmBZBmCZCmDZD  G d de;e          ZEdS )    )futures)AnyDictListcast)UUID)	overridesN)DEFAULT_DATABASEDEFAULT_TENANT	ComponentSystem)from_proto_metadatafrom_proto_update_metadatafrom_proto_segmentfrom_proto_segment_scopeto_proto_collectionto_proto_segment)CreateCollectionRequestCreateCollectionResponseCreateDatabaseRequestCreateDatabaseResponseCreateSegmentRequestCreateSegmentResponseCreateTenantRequestCreateTenantResponseCountCollectionsRequestCountCollectionsResponseDeleteCollectionRequestDeleteCollectionResponseDeleteSegmentRequestDeleteSegmentResponseGetCollectionsRequestGetCollectionsResponseGetCollectionSizeRequestGetCollectionSizeResponse GetCollectionWithSegmentsRequest!GetCollectionWithSegmentsResponseGetDatabaseRequestGetDatabaseResponseGetSegmentsRequestGetSegmentsResponseGetTenantRequestGetTenantResponseResetStateResponseUpdateCollectionRequestUpdateCollectionResponseUpdateSegmentRequestUpdateSegmentResponse)SysDBServiceradd_SysDBServicer_to_server)Empty)
CollectionMetadataSegmentSegmentScopec                       e Zd ZU dZej        ed<   eed<   i Ze	e
ef         ed<   i Ze	e
ee
         f         ed<   i Ze	e
e	e
e	e
ef         f         f         ed<   i Ze	e
e	e
ef         f         ed<   def fd	Zed( fd            Zed( fd            Zed( fd            Z ed          dedej        d
efd            Z ed          dedej        d
efd            Z ed          dedej        d
e fd            Z! ed          de"dej        d
e#fd            Z$ ed          de%dej        d
e&fd            Z'dedej        d
e&fdZ( ed          de)dej        d
e*fd            Z+ ed          de,dej        d
e-fd            Z. ed          de/dej        d
e0fd            Z1 ed          de2dej        d
e3fd            Z4 ed          de5dej        d
e6fd            Z7 ed          de8dej        d
e9fd            Z: ed          de;dej        d
e<fd             Z= ed          de>dej        d
e?fd!            Z@ ed          deAdej        d
eBfd"            ZC ed          deDdej        d
eEfd#            ZF ed          deGdej        d
eHfd$            ZId%eJd&eKjL        d
dfd'ZM xZNS ))GrpcMockSysDBzA mock sysdb implementation that can be used for testing the grpc client. It stores
    state in simple python data structures instead of a database._server_server_port	_segments_collection_to_segments$_tenants_to_databases_to_collections_tenants_to_database_to_idsystemc                     |j                             d          | _        t                                          |          S )Nchroma_server_grpc_port)settingsrequirer=   super__init__)selfrB   	__class__s     Y/root/projects/butler/venv/lib64/python3.11/site-packages/chromadb/db/impl/grpc/server.pyrH   zGrpcMockSysDB.__init__H   s4    "O334MNNww'''    returnNc                 <   t          j        t          j        d                    | _        t          | | j                   | j                            d| j                    | j                                         t                                                      S )N
   )max_workersz[::]:)
grpcserverr   ThreadPoolExecutorr<   r4   add_insecure_portr=   startrG   rI   rJ   s    rK   rU   zGrpcMockSysDB.startL   s|    {7#="#M#M#MNN#D$,777&&'Bt/@'B'BCCCww}}rL   c                 x    | j                             d            t                                                      S N)r<   stoprG   rV   s    rK   rY   zGrpcMockSysDB.stopT   s+    $ww||~~rL   c                     i | _         i | _        i | j        t          <   i | j        t                   t          <   i | j        t          <   t          d          | j        t                   t          <   t                                                      S )Nr   )int)r>   r@   r   r
   rA   r   rG   reset_staterV   s    rK   r\   zGrpcMockSysDB.reset_stateY   sq    461DF1.AVX1.ABRS:<'7LPUVKKK'78HIww""$$$rL   F)check_signaturerequestcontextc                 t   |j         }|j        }|| j        vr)|                    t          j        j        d| d           || j        |         v r)|                    t          j        j        d| d           i | j        |         |<   t          |j	                  | j
        |         |<   t                      S )NTenant 
 not found	Database  already existshex)tenantnamer@   abortrQ   
StatusCode	NOT_FOUNDALREADY_EXISTSr   idrA   r   )rI   r^   r_   rg   databases        rK   CreateDatabasezGrpcMockSysDB.CreateDatabased   s     <BBBMM$/35Qv5Q5Q5QRRRt@HHHMM.0UH0U0U0U   GI1&9(C<@WZ<P<P<P'/9%'''rL   c                 l   |j         }|j        }|| j        vr)|                    t          j        j        d| d           || j        |         vr)|                    t          j        j        d| d           | j        |         |         }t          t          j
        |j        ||                    S )Nra   rb   rc   )rm   rh   rg   )rn   )rg   rh   r@   ri   rQ   rj   rk   rA   r)   protoDatabaserf   )rI   r^   r_   rg   rn   rm   s         rK   GetDatabasezGrpcMockSysDB.GetDatabaset   s     <BBBMM$/35Qv5Q5Q5QRRR4DVLLLMM$/35U5U5U5UVVV,V4X>"^rvHVLLL
 
 
 	
rL   c                     |j         }|| j        v r)|                    t          j        j        d| d           i | j        |<   i | j        |<   t                      S )Nra   rd   )rh   r@   ri   rQ   rj   rl   rA   r   rI   r^   r_   rg   s       rK   CreateTenantzGrpcMockSysDB.CreateTenant   sm     T>>>MM.0Q&0Q0Q0Q   =?1&924'/#%%%rL   c                     |j         }|| j        vr)|                    t          j        j        d| d           t          t          j        |                    S )Nra   rb   rh   )rg   )	rh   r@   ri   rQ   rj   rk   r-   rq   Tenantru   s       rK   	GetTenantzGrpcMockSysDB.GetTenant   se     BBBMM$/35Qv5Q5Q5QRRR <V,,,
 
 
 	
rL   c                 V    t          |j                  }|                     ||          S rX   )r   segmentCreateSegmentHelper)rI   r^   r_   r|   s       rK   CreateSegmentzGrpcMockSysDB.CreateSegment   s)     %W_55''999rL   r|   c                     |d         j         | j        v r/|                    t          j        j        d|d          d           || j        |d         j         <   t                      S )Nrm   Segment rd   )rf   r>   ri   rQ   rj   rl   r   )rI   r|   r_   s      rK   r}   z!GrpcMockSysDB.CreateSegmentHelper   si     4=..MM.974=999   -4wt}()$&&&rL   c                     |j         }|| j        v r| j        |= t                      S |                    t          j        j        d| d           d S )Nr   rb   )rm   r>   r!   ri   rQ   rj   rk   )rI   r^   r_   id_to_deletes       rK   DeleteSegmentzGrpcMockSysDB.DeleteSegment   sd     z4>))|,(***MM)+Nl+N+N+N    rL   c                 (   |                     d          rt          |j                  nd }|                     d          r|j        nd }|                     d          rt	          |j                  nd }t          |j                  }g }| j                                        D ]S}|r|d         |k    r|r|d         |k    r |r|d         |k    r/|r|d         |k    r>|	                    |           Tt          d |D                       S )Nrm   re   typescope
collectionc                 ,    g | ]}t          |          S  r   .0r|   s     rK   
<listcomp>z-GrpcMockSysDB.GetSegments.<locals>.<listcomp>   s!    NNNG&w//NNNrL   )segments)HasFieldr   rm   r   r   r   r   r>   valuesappendr+   )	rI   r^   r_   	target_idtarget_typetarget_scopetarget_collectionfound_segmentsr|   s	            rK   GetSegmentszGrpcMockSysDB.GetSegments   sZ    -4,<,<T,B,BLDWZ((((	&-&6&6v&>&>HgllD (($W]333 	
 !W%7888~,,.. 		+ 		+G WT]i77 wv+==  0L @ @  W\%:>O%O%O!!'****"NN~NNN
 
 
 	
rL   c                    t          |j                  }|j        | j        vr+|                    t
          j        j        d| d           d S | j        |j                 }|                    d          rUt          t          t          t          f         |d                   }|d         i |d<   |                     ||j                   |                    d          r|j        ri |d<   t!                      S )Nr   rb   metadatareset_metadata)r   rm   rf   r>   ri   rQ   rj   rk   r   r   r   strr   _merge_metadatar   r   r2   )rI   r^   r_   id_to_updater|   targets         rK   UpdateSegmentzGrpcMockSysDB.UpdateSegment   s    GJ''4>11MM)+Nl+N+N+N     n\%56G
++ ?d38ngj.ABB:&.*,GJ'$$VW-=>>> 011 )g6L )&(
#(***rL   c                    |j         |j        }|j        }|| j        vr)|                    t
          j        j        d| d           || j        |         vr)|                    t
          j        j        d| d           | j                                        D ]\  }}|                                D ]\  }}|j	        |v r||j        k    s||j        k    r4|                    t
          j        j
        d|j	         d| d|            X|j        s3|                    t
          j        j
        d|j	         d| d|            | j        |         |         }	fd|	                                D             }
t          |
          dk    sJ t          |
          d	k    rV|j        r&|
d	         }t          t          |          d
          S |                    t
          j        j
        d d           t!          j        |j                  }t'          |j	                  }t)          ||j         |d t+          |j                  |j        ||d		  	        }g }|j        D ]}t3          |          }|d         j        | j        v rX|D ]&}|                     t;          |          |           '|                    t
          j        j
        d|d          d           |                     ||           |                    |d         j                   ||	|j	        <   | d| d|j	         }|| j         |<   t          t          |          d          S )Nra   rb   rc   Collection z already exists in tenant z
 database c                 ,    g | ]}|d          k    |S rx   r   )r   ccollection_names     rK   r   z2GrpcMockSysDB.CreateCollection.<locals>.<listcomp>  s'    SSSai?6R6R16R6R6RrL      r   F)r   createdrd   re   )	rm   rh   configuration_jsonserialized_schemar   	dimensionrn   rg   versionrm   )rm   r   :T)!rh   rg   rn   r@   ri   rQ   rj   rk   itemsrm   rl   get_or_creater   lenr   r   jsonloadsconfiguration_json_strr   r6   r   r   r   r   r   rf   r>   r   r    r}   r   r?   )rI   r^   r_   rg   rn   search_tenant	databasessearch_databasesearch_collectionscollectionsmatchesexisting_collectionr   rm   new_collectionsegments_addedsegment_protor|   scollection_unique_keyr   s                       @rK   CreateCollectionzGrpcMockSysDB.CreateCollection   s    ",#BBBMM$/35Qv5Q5Q5QRRR4DVLLLMM$/35U5U5U5UVVV 6<<>>	 	 
7@7H7H  3!3:!333%77*g.>>> O:z'*zzP]zzixzz    %2    O:z'*zzP]zzixzz  & ?GQSSSSk0022SSS7||q    w<<!$ &-aj#/23FGG!    MM.>o>>>  
 "Z(FGGgj!!!#1"()9::'

 

 

 $- 	5 	5M(77Gt} DN22' L LA&&';q'A'A'A7KKKKO2=wt}===   $$Wg666!!'$-"34444"0GJ#) C CH C Cwz C C>L$%:;'*>::
 
 
 	
rL   c                    |j         }|j        }|j        }|| j        vr)|                    t
          j        j        d| d           || j        |         vr)|                    t
          j        j        d| d           | j        |         |         }||v r7||= | d| d| }| j        |         }|r|D ]
}	| j	        |	= t                      S |                    t
          j        j        d| d           d S )Nra   rb   rc   r   r   )rm   rg   rn   r@   ri   rQ   rj   rk   r?   r>   r   )
rI   r^   r_   collection_idrg   rn   r   r   segment_ids
segment_ids
             rK   DeleteCollectionzGrpcMockSysDB.DeleteCollectionH  s:     
#BBBMM$/35Qv5Q5Q5QRRR4DVLLLMM$/35U5U5U5UVVV?GQK''M*'-$J$J$J$J=$J$J!67LMK 3"- 3 3Jz22+---MM)+R+R+R+R    rL   c           
         |                     d          rt          |j                  nd }|                     d          r|j        nd }i }| j                                        D ]z\  }}|                                D ]`\  }}	|j        dk    r||j        k    r|j        dk    r||j        k    r3|                    |	           t          d| d| d|	            a{g }
|
                                D ]5}|r|d         |k    r|r|d         |k    r |
                    |           6t          d |
D             	          S )
Nrm   re   rh    Tenant: , Database: , Collections: c                 ,    g | ]}t          |          S r   )r   )r   r   s     rK   r   z0GrpcMockSysDB.GetCollections.<locals>.<listcomp>{  s.       4>#J//  rL   )r   )r   r   rm   rh   r@   r   rg   rn   updateprintr   r   r#   )rI   r^   r_   r   target_nameallCollectionsrg   r   rn   r   found_collectionsr   s               rK   GetCollectionszGrpcMockSysDB.GetCollectionsa  s    -4,<,<T,B,BLDWZ((((	&-&6&6v&>&>HgllD!%!J!P!P!R!R 		 		FI)2):):  %+>R''Fgn,D,D#r))h':J.J.J%%k222YvYY8YYKYY    (//11 	1 	1J Z-:: z&1[@@$$Z0000% BS  
 
 
 	
rL   c                     t          |j        |j                  }|                     ||          }t	          t          |j                            S )N)rg   rn   )count)r"   rg   rn   r   r   r   r   )rI   r^   r_   r   s       rK   CountCollectionszGrpcMockSysDB.CountCollections  sV     (>%
 
 
 ))'7;;'c+2I.J.JKKKKrL   c                 "    t          d          S )Nr   )total_records_post_compaction)r%   rI   r^   r_   s      rK   GetCollectionSizezGrpcMockSysDB.GetCollectionSize  s     )*+
 
 
 	
rL   c           
          i } j                                         D ]L\  }}|                                D ]2\  }}|                    |           t          d| d| d|            3M|                    |j        d           }|.|                    t          j        j	        d|j         d           |j
         d|j         d|j         }	 fd j        |	         D             }
d |
D             t          j        t          j        t          j        hk    r+|                    t          j        j        d	| d
|
            t%          t'          |          d |
D                       S )Nr   r   r   zCollection with id rb   r   c                 *    g | ]}j         |         S r   )r>   )r   rm   rI   s     rK   r   z;GrpcMockSysDB.GetCollectionWithSegments.<locals>.<listcomp>  s0     
 
 
 N2
 
 
rL   c                     h | ]
}|d          S )r   r   r   s     rK   	<setcomp>z:GrpcMockSysDB.GetCollectionWithSegments.<locals>.<setcomp>  s    555GG555rL   z#Incomplete segments for collection z: c                 ,    g | ]}t          |          S r   r   r   s     rK   r   z;GrpcMockSysDB.GetCollectionWithSegments.<locals>.<listcomp>  s!    HHHG&w//HHHrL   )r   r   )r@   r   r   r   getrm   ri   rQ   rj   rk   rg   rn   r?   r9   METADATARECORDVECTORINTERNALr'   r   )rI   r^   r_   r   rg   r   rn   r   r   r   r   s   `          rK   GetCollectionWithSegmentsz'GrpcMockSysDB.GetCollectionWithSegments  s    !%!J!P!P!R!R 	 	FI)2):):  %+%%k222YvYY8YYKYY   
 $''
D99
MM)+W+W+W+W    EE:#6EEEE 	
 
 
 
23HI
 
 
 65H555!:
 
 

 MM(NjNNHNN  
 1*:66HHxHHH
 
 
 	
rL   c                    t          |j                  }i }| j                                        D ]*\  }}|                                D ]\  }}|j        |v r|}+|j        |vr+|                    t          j        j        d| d           d S ||j                 }	|	                    d          r
|j
        |	d<   |	                    d          r
|j        |	d<   |	                    d          rAt          |j                  }
d }|
#i }|
                                D ]\  }}||||<   ||	d<   n!|	                    d          r|j        ri |	d<   t                      S )Nr   rb   rh   r   r   r   )r   rm   r@   r   rf   ri   rQ   rj   rk   r   rh   r   r   r   r   r0   )rI   r^   r_   r   r   rg   r   rn   maybe_collectionsr   update_metadatacleaned_metadatakeyvalues                 rK   UpdateCollectionzGrpcMockSysDB.UpdateCollection  s    GJ''!%!J!P!P!R!R 	4 	4FI/8/@/@ 4 4++#'888"3K4 ;..MM)+Q+Q+Q+Q     %\%56J'' 2%,\
6",, <*1*;
;'
++ 0
 #=W=M"N"N#' ".')$&5&;&;&=&= : :
U ,49,S1)9
:&&!!"233 0) 0-/Jz*+---rL   c                 F    |                                   t                      S rX   )r\   r.   r   s      rK   
ResetStatezGrpcMockSysDB.ResetState  s"     	!###rL   r   sourcec                 ,   t          t          t          t          f         |          }t          t          t          t          f         t	          |                    }|                    |           |                                D ]\  }}|||v r||= d S rX   )r   r   r   r   r   r   r   )rI   r   r   target_metadatasource_metadatar   r   s          rK   r   zGrpcMockSysDB._merge_metadata  s    tCH~v66tCH~/I&/Q/QRR///)//11 	) 	)JC}#C(	) 	)rL   )rM   N)O__name__
__module____qualname____doc__rQ   Server__annotations__r[   r>   r   r   r8   r?   r   r@   r6   rA   r   r   rH   r	   rU   rY   r\   r   ServicerContextr   ro   r(   r)   rs   r   r   rv   r,   r-   rz   r   r   r~   r}   r    r!   r   r*   r+   r   r1   r2   r   r   r   r   r   r   r   r"   r#   r   r   r   r   r$   r%   r   r&   r'   r   r/   r0   r   r5   r.   r   r7   rq   UpdateMetadatar   __classcell__)rJ   s   @rK   r;   r;   ;   s        E E [$&ItCL!&&&46T#tCy.1666 	 )$T#tCO,,--+    >@S$sDy/%9 :???(v ( ( ( ( ( (      Y      Y % % % % % Y% Yu%%%(,(7;7K(	( ( ( &%( Yu%%%
)
484H
	
 
 
 &%
 Yu%%%
&*
&595I
&	
& 
& 
& &%
& Yu%%%
'
262F
	
 
 
 &%
 Yu%%%:+:6:6J:	: : : &%:	'	')-)=	'		' 	' 	' 	' Yu%%%
+
6:6J
	
 
 
 &%
 Yu%%%
)
484H
	
 
 
 &%
6 Yu%%%+++6:6J+	+ + + &%+& Yu%%%X
.X
9=9MX
	!X
 X
 X
 &%X
t Yu%%%.9=9M	!   &%0 Yu%%%
,
7;7K
	
 
 
 &%
< Yu%%%L.L9=9ML	!L L L &%L Yu%%%
/
:>:N
	"
 
 
 &%
 Yu%%%#
7#
BFBV#
	*#
 #
 #
 &%#
J Yu%%%'..'.9=9M'.	!'. '. '. &%'.R Yu%%%$$'+';$	$ $ $ &%$)h )8L )QU ) ) ) ) ) ) ) )rL   r;   )F
concurrentr   typingr   r   r   r   uuidr   r	   r   chromadb.configr
   r   r   r   chromadb.proto.convertr   r   r   r   r   r   chromadb.proto.chroma_pb2rq   
chroma_pb2chromadb.proto.coordinator_pb2r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   #chromadb.proto.coordinator_pb2_grpcr3   r4   rQ   google.protobuf.empty_pb2r5   chromadb.typesr6   r7   r8   r9   r;   r   rL   rK   <module>r     s         ( ( ( ( ( ( ( ( ( ( ( (              O O O O O O O O O O O O                * ) ) ) ) ) ) ) )                                                                                                                                   B         + + + + + + F F F F F F F F F F F Fv) v) v) v) v)M9 v) v) v) v) v)rL   