
    ZiK#                        d dl Z d dlZd dlmZmZmZmZmZ d dlZd dl	m	Z	 d dl
mZmZmZ d dlmZ d dlmZ d dlmZ d dlmZmZmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dl m!Z!m"Z"m#Z#m$Z$m%Z% d dl&m'Z' dee         dee         fdZ(dee         dee)         fdZ* ed          Z+ ed          Z, G d de          Z-dS )    N)CallableDictListOptionalTypeVar)	overrides)	GetResultMetadataQueryResult)System)Executor)Scan)	CountPlanGetPlanKNNPlan)convert)QueryExecutorStub)DistributedSegmentManager)OtelInterceptor)RetryCallStateRetryingstop_after_attemptwait_exponential_jitterretry_if_exception)Spanmetadatareturnc                     | sdS i }|                                  D ]\  }}|                    d          s|||<    t          |          dk    rdS |S )z[Remove any chroma-specific metadata keys that the client shouldn't see from a metadata map.Nzchroma:r   )items
startswithlen)r   resultkvs       d/root/projects/butler/venv/lib64/python3.11/site-packages/chromadb/execution/executor/distributed.py_clean_metadatar&      sj     tF    1||I&& 	F1I
6{{atM    c                 <    | rd| v rt          | d                   S dS )z-Retrieve the uri (if any) from a Metadata mapz
chroma:uriN)strr   s    r%   _urir+   &   s.      +LH,,8L)***4r'   IOc                   0    e Zd ZU ej        ed<   eeef         ed<   e	ed<   e
ed<   e
ed<   def fdZdeeegef                  d	ed
efdZeded
e
fd            Zeded
efd            Zeded
efd            Zded
ee         fdZded
efdZ xZS )DistributedExecutor_mtx_grpc_stub_pool_manager_request_timeout_seconds_query_replication_factorsystemc                 B   t                                          |           t          j                    | _        i | _        |                     t                    | _        |j	                            d          | _
        |j	                            d          | _        d S )N$chroma_query_request_timeout_secondschroma_query_replication_factor)super__init__	threadingLockr0   r1   requirer   r2   settingsr3   r4   )selfr5   	__class__s     r%   r:   zDistributedExecutor.__init__:   s       N$$	!%>??(.(?(?2)
 )
% *0)@)@-*
 *
&&&r'   funcsargsr   c                 ~   d}ddt           ddffd}t          t          d          t          dd          d	t	          d
           |          D ]\}                                 d|5   ||t          |          z           |          cddd           c S # 1 swxY w Y   |dz  }]t          d          )z
        Retry a list of functions in a round-robin fashion until one of them succeeds.

        funcs: List of functions to retry
        args: Arguments to pass to each function

        r   N_r   c                 F    ddl m} ||                    d          d S d S )Nr   )tracerzWaiting to retry RPC) chromadb.telemetry.opentelemetryrF   
start_span)rD   rF   
sleep_spans     r%   before_sleepz<DistributedExecutor._round_robin_retry.<locals>.before_sleepQ   s@     @????? !#../EFF


 "!r'      g?)jitterTc                     t          | t          j                  o5|                                 t          j        j        t          j        j        fv S N)
isinstancegrpcRpcErrorcode
StatusCodeUNAVAILABLEUNKNOWN)xs    r%   <lambda>z8DistributedExecutor._round_robin_retry.<locals>.<lambda>`   s:    *Q66 WFFHH!<do>U VV r'   )stopwaitreraiseretryrJ      z0Unreachable code error - should never reach here)r   r   r   r   r   endr!   	Exception)r?   rA   rB   attempt_countrJ   attemptrI   s         @r%   _round_robin_retryz&DistributedExecutor._round_robin_retryF   s`    %)
	GN 	Gt 	G 	G 	G 	G 	G 	G  #A&&(S999$W W  &	
 	
 	
 	 	G %   !
 ? ?8u]SZZ78>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?QMM JKKKs   /!BB#	&B#	planc                                            |j                  } fd|D             }                     |t          j        |                    }t          j        |          S )Nc                 D    g | ]}                     |          j        S  )	_get_stubCount.0endpointr?   s     r%   
<listcomp>z-DistributedExecutor.count.<locals>.<listcomp>s   s(    PPP(t~~h//5PPPr'   )_get_grpc_endpointsscanra   r   to_proto_count_planfrom_proto_count_result)r?   rb   	endpointscount_funcscount_results   `    r%   countzDistributedExecutor.countp   si    ,,TY77	PPPPiPPP..4T::
 
 .|<<<r'   c           	                                |j                  } fd|D             }                     |t          j        |                    }t          j        |          }d |D             }|j        j        rd |D             nd }|j        j        rd |D             nd }|j        j	        rd |D             nd }	|j        j
        rd |D             nd }
t          ||||	d |
|j        j                  S )Nc                 D    g | ]}                     |          j        S re   )rf   Getrh   s     r%   rk   z+DistributedExecutor.get.<locals>.<listcomp>|   (    LLLhT^^H--1LLLr'   c                     g | ]
}|d          S )idre   ri   records     r%   rk   z+DistributedExecutor.get.<locals>.<listcomp>   s    222vd|222r'   c                     g | ]
}|d          S )	embeddingre   rz   s     r%   rk   z+DistributedExecutor.get.<locals>.<listcomp>   s    777VVK 777r'   c                     g | ]
}|d          S )documentre   rz   s     r%   rk   z+DistributedExecutor.get.<locals>.<listcomp>   s    666FVJ666r'   c                 8    g | ]}t          |d                    S r*   r+   rz   s     r%   rk   z+DistributedExecutor.get.<locals>.<listcomp>   s%    <<<&T&$%%<<<r'   c                 8    g | ]}t          |d                    S r*   r&   rz   s     r%   rk   z+DistributedExecutor.get.<locals>.<listcomp>   s%    GGGV_VJ/00GGGr'   )ids
embeddings	documentsurisdata	metadatasincluded)rl   rm   ra   r   to_proto_get_planfrom_proto_get_result
projectionr}   r   urir   r	   r   )r?   rb   rp   	get_funcs
get_resultrecordsr   r   r   r   r   s   `          r%   getzDistributedExecutor.gety   sT   ,,TY77	LLLL)LLL	,,Y8QRV8W8WXX
/
;;22'222 (77w7777 	 '66g6666 	 "<<G<<<< 	 'GGwGGGG 	 !_-
 
 
 	
r'   c           
      *                          |j                  } fd|D             }                     |t          j        |                    }t          j        |          }d |D             }|j        j        rd |D             nd }|j        j        rd |D             nd }|j        j	        rd |D             nd }	|j        j
        rd |D             nd }
|j        j        rd |D             nd }t          ||||	d |
||j        j                  S )	Nc                 D    g | ]}                     |          j        S re   )rf   KNNrh   s     r%   rk   z+DistributedExecutor.knn.<locals>.<listcomp>   rw   r'   c                 &    g | ]}d  |D             S )c                 *    g | ]}|d          d         S )r{   ry   re   rz   s     r%   rk   z6DistributedExecutor.knn.<locals>.<listcomp>.<listcomp>   s"    ===6x &===r'   re   ri   r   s     r%   rk   z+DistributedExecutor.knn.<locals>.<listcomp>   s'    UUU'==W===UUUr'   c                 &    g | ]}d  |D             S )c                 *    g | ]}|d          d         S )r{   r}   re   rz   s     r%   rk   z6DistributedExecutor.knn.<locals>.<listcomp>.<listcomp>   s"    EEE6!+.EEEr'   re   r   s     r%   rk   z+DistributedExecutor.knn.<locals>.<listcomp>   s7        FEWEEE  r'   c                 &    g | ]}d  |D             S )c                 *    g | ]}|d          d         S )r{   r   re   rz   s     r%   rk   z6DistributedExecutor.knn.<locals>.<listcomp>.<listcomp>   s"    DDD&!*-DDDr'   re   r   s     r%   rk   z+DistributedExecutor.knn.<locals>.<listcomp>   s7        EDGDDD  r'   c                 &    g | ]}d  |D             S )c                 D    g | ]}t          |d          d                   S r{   r   r   rz   s     r%   rk   z6DistributedExecutor.knn.<locals>.<listcomp>.<listcomp>   s*    JJJfX&z233JJJr'   re   r   s     r%   rk   z+DistributedExecutor.knn.<locals>.<listcomp>   s7        KJ'JJJ  r'   c                 &    g | ]}d  |D             S )c                 D    g | ]}t          |d          d                   S r   r   rz   s     r%   rk   z6DistributedExecutor.knn.<locals>.<listcomp>.<listcomp>   s*    UUU6!1*!=>>UUUr'   re   r   s     r%   rk   z+DistributedExecutor.knn.<locals>.<listcomp>   s7        VUWUUU  r'   c                 &    g | ]}d  |D             S )c                     g | ]
}|d          S )distancere   rz   s     r%   rk   z6DistributedExecutor.knn.<locals>.<listcomp>.<listcomp>   s    777VfZ 777r'   re   r   s     r%   rk   z+DistributedExecutor.knn.<locals>.<listcomp>   s'    OOOG77w777OOOr'   )r   r   r   r   r   r   	distancesr   )rl   rm   ra   r   to_proto_knn_planfrom_proto_knn_batch_resultr   r}   r   r   r   rankr   r   )r?   rb   rp   	knn_funcs
knn_resultresultsr   r   r   r   r   r   s   `           r%   knnzDistributedExecutor.knn   s   ,,TY77	LLLL)LLL	,,Y8QRV8W8WXX
5jAAUUWUUU (	  &   
  	 '	  &   
  	 "	  &   
  	 '	  &   
  	 #OOwOOOO 	 !_-	
 	
 	
 		
r'   rm   c                 x    | j                             |j        | j                  }t	          j        |           |S rN   )r2   get_endpointsr{   r4   randomshuffle)r?   rm   	grpc_urlss      r%   rl   z'DistributedExecutor._get_grpc_endpoints   s=     M//K7
 
	 	y!!!r'   grpc_urlc                    | j         5  || j        vrPt          j        |ddg          }t	                      g}t          j        |g|R  }t          |          | j        |<   | j        |         cd d d            S # 1 swxY w Y   d S )N)zgrpc.max_concurrent_streamsi  )zgrpc.max_receive_message_lengthi H)options)r0   r1   rP   insecure_channelr   intercept_channelr   )r?   r   channelinterceptorss       r%   rf   zDistributedExecutor._get_stub   s    Y 	2 	2t333/=E   !0 1 120H<HHH1B71K1K$X.'1	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2 	2s   A&A;;A?A?)__name__
__module____qualname__r;   r<   __annotations__r   r)   r   r   intr   r:   r   r   r,   r-   ra   r   r   rs   r   r	   r   r   r   r   r   rl   rf   __classcell__)r@   s   @r%   r/   r/   3   s        
.#001111''''!!!!""""

v 

 

 

 

 

 

(LXqc1f-=(> (La (LA (L (L (L (LT =) = = = = Y= %
 %
I %
 %
 %
 Y%
N 7
 7
K 7
 7
 7
 Y7
r c    2# 2*; 2 2 2 2 2 2 2 2r'   r/   ).r;   r   typingr   r   r   r   r   rP   r   chromadb.api.typesr	   r
   r   chromadb.configr   $chromadb.execution.executor.abstractr   &chromadb.execution.expression.operatorr   "chromadb.execution.expression.planr   r   r   chromadb.protor   &chromadb.proto.query_executor_pb2_grpcr   )chromadb.segment.impl.manager.distributedr   %chromadb.telemetry.opentelemetry.grpcr   tenacityr   r   r   r   r   opentelemetry.tracer   r&   r)   r+   r,   r-   r/   re   r'   r%   <module>r      s$        : : : : : : : : : : : : : :        ? ? ? ? ? ? ? ? ? ? " " " " " " 9 9 9 9 9 9 7 7 7 7 7 7 J J J J J J J J J J " " " " " " D D D D D D O O O O O O A A A A A A              % $ $ $ $ $
hx0 
Xh5G 
 
 
 
8H% (3-     GCLLGCLL2 2 2 2 2( 2 2 2 2 2r'   