
    Zi}              	          d dl mZmZmZmZmZmZ d dlZd dlm	Z	m
Z
mZmZ d dlmZmZ d dlmZ d dlZd dlmZ  G d ded	
          Z G d ded	
          Z G d ded
          ZdedefdZdeeef         defdZdedefdZdedeeef         fdZ G d ded	
          Zdeeef         defdZ G d ded	
          Zdeeef         defdZ G d d ed	
          Z d!e
de fd"Z!d!eeef         de fd#Z"deeef         de fd$Z#	 dMde d!ee
         defd%Z$	 dMde d!ee
         deeef         fd&Z%	 dMded'ee         defd(Z& G d) d*ed	
          Z'deeef         de'fd+Z( G d, d-ed	
          Z)deeef         de)fd.Z* G d/ d0ed	
          Z+d!e
de+fd1Z,d!ede+fd2Z-de+defd3Z.de+deeef         fd4Z/d5ede+fd6Z0deeef         de+fd7Z1d8ed9e'defd:Z2d;ed<e)defd=Z3d>ed?edefd@Z4dAedBe+defdCZ5dDee         dEee         ddfdFZ6dDee         dGeeeef                  ddfdHZ7	 	 	 	 	 	 dNdLZ8dS )O    )	TypedDictDictAnyOptionalcastget_argsN)SpaceCollectionMetadataUpdateMetadataEmbeddingFunction)known_embedding_functionsregister_embedding_function)	cpu_count)Schemac                   `    e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   d	S )
HNSWConfigurationspaceef_constructionmax_neighbors	ef_searchnum_threads
batch_sizesync_thresholdresize_factorN__name__
__module____qualname__r	   __annotations__intfloat     b/root/projects/butler/venv/lib64/python3.11/site-packages/chromadb/api/collection_configuration.pyr   r      i         LLLNNNOOOr#   r   F)totalc                   j    e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
S )SpannConfigurationsearch_nprobewrite_nprober   r   r   r   reassign_neighbor_countsplit_thresholdmerge_thresholdNr   r   r   r    r   r	   r"   r#   r$   r(   r(      x         LLLNNN    r#   r(   c                   R    e Zd ZU ee         ed<   ee         ed<   ee         ed<   dS )CollectionConfigurationhnswspannembedding_functionN)r   r   r   r   r   r   r(   r   r"   r#   r$   r1   r1   *   sH         
$
%%%%&'''' !2333333r#   r1   Tconfig_json_strreturnc                 H    t          j        |           }t          |          S N)jsonloads'load_collection_configuration_from_json)r5   config_json_maps     r$   +load_collection_configuration_from_json_strr=   0   s!     j11O2?CCCr#   r<   c           	      R   |                      d          $|                      d          t          d          d }d }d }|                      d          t          t          | d                   }|                      d          t          t          | d                   }|                      d          | d         }|d         dk    rt          j        dt          d	           d }n	 |d
         }n # t          $ r t          d|           w xY w	 t          |         }n!# t          $ r t          d| d          w xY w	 |
                    |d                   }n8# t          $ r)}t          d|d
          d|d          d|           d }~ww xY wd }t          |||          S )Nr3   r2   &hnsw and spann cannot both be providedr4   typelegacy legacy embedding function config   
stacklevelnamez-Embedding function name not found in config: zEmbedding function zO not found. Add @register_embedding_function decorator to the class definition.configz#Could not build embedding function z from config z: r2   r3   r4   )get
ValueErrorr   r   r(   warningswarnDeprecationWarningKeyErrorr   build_from_config	Exceptionr1   )r<   hnsw_configspann_config	ef_configefef_namees          r$   r;   r;   8   s;    	G$$0''3ABBBKLI 6"".,of.EFF7##/.0HII /00<#$89	V((M2"   
 BB#F+    OIOO  .w7     C'  C  C  C  )))H*=>>    t)F:KttZcdlZmttqrtt  
 "   s0   )C2 2DD! !D?E 
F)$FFrG   c                 D    t          j        t          |                     S r8   )r9   dumps collection_configuration_to_json)rG   s    r$   $collection_configuration_to_json_strrZ   r   s    :6v>>???r#   c                    t          | t                    r@|                     d          }|                     d          }|                     d          }n	 |                     d          j        }n# t
          $ r d }Y nw xY w	 |                     d          j        }n# t
          $ r d }Y nw xY w	 |                     d          j        }n# t
          $ r d }Y nw xY wd }|;	 t          t          |          }n$# t          $ r}t          d|           d }~ww xY w|;	 t          t          |          }n$# t          $ r}t          d|           d }~ww xY w|d }ddi}|	 |
                                rddi}nF|                                d|                                d	}t          t          |                     n<# t          $ r/}t          j        d
| t"          d           d }ddi}Y d }~nd }~ww xY w|||dS )Nr2   r3   r4   not a valid hnsw config: not a valid spann config: r@   rA   knownrF   r@   rG   "legacy embedding function config: rC   rD   rH   )
isinstancedictrI   get_parametervaluerJ   r   r   rP   r(   	is_legacyrF   
get_configr   r@   rK   rL   rM   )rG   rQ   rR   rT   rS   rV   s         r$   rY   rY   v   s   &$ jj((zz'**ZZ,--	 ..v66<KK 	 	 	KKK		 !//88>LL 	  	  	 LLL	 	%%&:;;ABB 	 	 	BBB	 (,I	>0+>>KK 	> 	> 	><<<===	>	? 2LAALL 	? 	? 	?=!==>>>	? 
zX&		~	+||~~ 6#X.		 GGII# mmoo 	
 ,DHH555 	+ 	+ 	+M8Q88"   
 B*IIIIII	+ '  s   A2 2B BB   B/.B/3C CC%C; ;
DDD"D8 8
EEE'AG 
H %G;;H c                   `    e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   d	S )
CreateHNSWConfigurationr   r   r   r   r   r   r   r   Nr   r"   r#   r$   rh   rh      r%   r#   rh   json_mapc                 P   i }d| v r6| d         }|t          t                    v r||d<   nt          d|           d| v r| d         |d<   d| v r| d         |d<   d| v r| d         |d<   d| v r| d         |d<   d| v r| d         |d<   d| v r| d         |d<   d	| v r| d	         |d	<   |S )
Nr   not a valid space: r   r   r   r   r   r   r   r   r	   rJ   ri   rG   space_values      r$   !json_to_create_hnsw_configurationro      s    ')F(w'(5//)))F7OO@;@@AAAH$$$,->$? !("""*?";h&{3{   ( 7}x'5|8###+,<#= ("""*?";Mr#   c                   j    e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   d
S )CreateSpannConfigurationr)   r*   r   r   r   r   r+   r,   r-   Nr.   r"   r#   r$   rq   rq      r/   r#   rq   c                    i }d| v r| d         |d<   d| v r| d         |d<   d| v r6| d         }|t          t                    v r||d<   nt          d|           d| v r| d         |d<   d| v r| d         |d<   d| v r| d         |d<   |S )Nr)   r*   r   rk   r   r   r   rl   rm   s      r$   "json_to_create_spann_configurationrs      s     (*F("""*?";!!!).!9~(w'(5//)))F7OO@;@@AAAH$$$,->$? !h&{3{("""*?";Mr#   c                   R    e Zd ZU ee         ed<   ee         ed<   ee         ed<   dS )CreateCollectionConfigurationr2   r3   r4   N)r   r   r   r   rh   r   rq   r   r"   r#   r$   ru   ru      H         
*
++++,---- !2333333r#   ru   metadatac                      t          |           S )FCreate a CreateCollectionConfiguration from legacy collection metadata)9create_collection_configuration_from_legacy_metadata_dict)rw   s    r$   ?create_collection_configuration_from_legacy_collection_metadatar{     s     EXNNNr#   c           	          ddddddddd	}i }|                                  D ]\  }}||v r||||         <   t          |          }t          |          }t          |
          S )ry   r   r   r   r   r   r   r   r   )
hnsw:spacezhnsw:construction_efzhnsw:Mhnsw:search_efhnsw:num_threadshnsw:batch_sizehnsw:sync_thresholdhnsw:resize_factorr2   )itemsro   populate_create_hnsw_defaultsru   rw   
old_to_newri   rF   rd   rQ   s         r$   rz   rz   
  s    
  1!%)'/-	 	J H~~'' / /e:).HZ%&3H==K/<<K(k::::r#   c                 6   |                      d          $|                      d          t          d          t                      }|                      d          t          | d                   |d<   |                      d          t	          | d                   |d<   |                      d          b| d         }|d         dk    rt          j        dt          d	           n1t          |d
                  }|	                    |d                   |d<   |S )Nr2   r3   r?   r4   r@   rA   rB   rC   rD   rF   rG   )
rI   rJ   ru   ro   rs   rK   rL   rM   r   rO   )ri   resultrS   rT   s       r$   .load_create_collection_configuration_from_jsonr   #  s    ||F'HLL,A,A,MABBB*,,F ||F':8F;KLLv||G(<Xg=NOOw ||())512	V((M2"     +9V+<=B+-+?+?	(@S+T+TF'(Mr#   c                 F    t          j        t          | |                    S )zFConvert a CreateCollection configuration to a JSON-serializable string)r9   rX   'create_collection_configuration_to_json)rG   rw   s     r$   +create_collection_configuration_to_json_strr   B  s    
 :=fhOOPPPr#   c           	         d}|                      d          }|                      d          }|;	 t          t          |          }n$# t          $ r}t	          d|           d}~ww xY w|;	 t          t
          |          }n$# t          $ r}t	          d|           d}~ww xY w||t	          d          |                      d          d}dd	i}|||d
S 	 t          t          |                      d                    }|                                rdd	i}n|;|9||                     d          "t          |                                          }|,|                     d          |                                |d<   |,|                     d          |                                |d<   ||                     d          |	                                vr\t          j        d|                     d           d|                                 d|	                                 t          d           ||                     d          |	                                vr\t          j        d|                     d           d|                                 d|	                                 t          d           ||||                     d          |                     d          |	                                vr\t          j        d|                     d           d|                                 d|	                                 t          d           |                                d|                                d}t          t!          |                     n<# t          $ r/}t          j        d| t"          d           d}dd	i}Y d}~nd}~ww xY w|||d
S )zDConvert a CreateCollection configuration to a JSON-serializable dictNr2   r3   r\   r]   r?   r4   r@   rA   rH   r}   )r   r   zspace z is not supported by z. Supported spaces: rC   rD   r^   r_   r`   )rI   r   rh   rP   rJ   rq   r   re   default_spacesupported_spacesrK   rL   rF   UserWarningrf   r   r@   rM   )rG   rw   rS   rQ   rR   rV   rT   s          r$   r   r   K  s   
 (,I**V$$K::g&&L	>6DDKK 	> 	> 	><<<===	>	? 8,GGLL 	? 	? 	?=!==>>>	? <#;ABBBzz&''/X&	!"+
 
 	
B'#VZZ0D%E%EFF<<>> 8	2*II "|';#x||L'A'A'I"9@P@P@R@R"S"S"SK &;??7+C+C+K')'7'7'9'9G$'L,<,<W,E,E,M(*(8(8(:(:W% &??7++23F3F3H3HHHM G!9!9  G  GPRPWPWPYPY  G  Goq  pC  pC  pE  pE  G  G##$   
 '##G,,B4G4G4I4IIIM H!1!1'!:!:  H  HQSQXQXQZQZ  H  Hpr  qD  qD  qF  qF  H  H##$    # ((LL..:<<--R5H5H5J5JJJM Il!;!;  I  IRTRYRYR[R[  I  Iqs  rE  rE  rG  rG  I  I##$    		--// I
 (R111 ' ' '444	
 	
 	
 	

 X&						' '  sH   A 
A'A""A'-B 
B$BB$KN, ,
O%6%O  O%rT   c                    |                      d          |r|                                nd| d<   |                      d          d| d<   |                      d          d| d<   |                      d          d| d<   |                      d	          t                      | d	<   |                      d
          d| d
<   |                      d          d| d<   |                      d          d| d<   | S )z7Populate a CreateHNSW configuration with default valuesr   Nl2r   d   r      r   r   r   r   i  r   g333333?)rI   r   r   )rG   rT   s     r$   r   r     s    zz'"02<"**,,,wzz#$$,$' !zz/""*"$zz+&!{zz-  ( )}zz,'"|zz"##+#' zz/""*"%Mr#   c                   B    e Zd ZU eed<   eed<   eed<   eed<   eed<   dS )UpdateHNSWConfigurationr   r   r   r   r   N)r   r   r   r    r   r!   r"   r#   r$   r   r     sH         NNNOOOr#   r   c                     i }d| v r| d         |d<   d| v r| d         |d<   d| v r| d         |d<   d| v r| d         |d<   d| v r| d         |d<   |S )Nr   r   r   r   r   r"   ri   rG   s     r$   !json_to_update_hnsw_configurationr     s     ')Fh&{3{   ( 7}x'5|8###+,<#= ("""*?";Mr#   c                   $    e Zd ZU eed<   eed<   dS )UpdateSpannConfigurationr)   r   N)r   r   r   r    r   r"   r#   r$   r   r     s%         NNNNNr#   r   c                 F    i }d| v r| d         |d<   d| v r| d         |d<   |S )Nr)   r   r"   r   s     r$   "json_to_update_spann_configurationr     sB     (*F("""*?";h&{3{Mr#   c                   R    e Zd ZU ee         ed<   ee         ed<   ee         ed<   dS )UpdateCollectionConfigurationr2   r3   r4   N)r   r   r   r   r   r   r   r   r"   r#   r$   r   r     rv   r#   r   c                     dddddd}i }|                                  D ]\  }}||v r||||         <   t          |          }t          |          S )zGCreate an UpdateCollectionConfiguration from legacy collection metadatar   r   r   r   r   r~   r   r   r   r   r   r   r   r   r   s         r$   ?update_collection_configuration_from_legacy_collection_metadatar     |    
 &)'/- J H~~'' / /e:).HZ%&3H==K(k::::r#   c                     dddddd}i }|                                  D ]\  }}||v r||||         <   t          |          }t          |          S )zCCreate an UpdateCollectionConfiguration from legacy update metadatar   r   r   r   r   r   r   r   r   s         r$   ;update_collection_configuration_from_legacy_update_metadatar     r   r#   c                 H    t          |           }t          j        |          S )zFConvert an UpdateCollectionConfiguration to a JSON-serializable string)'update_collection_configuration_to_jsonr9   rX   )rG   	json_dicts     r$   +update_collection_configuration_to_json_strr   !  s!     8??I:i   r#   c                    |                      d          }|                      d          }|                      d          }|||i S |;	 t          t          |          }n$# t          $ r}t	          d|           d}~ww xY w|;	 t          t
          |          }n$# t          $ r}t	          d|           d}~ww xY wd}||                                rddi}np|                    |                                           |	                                d	|                                d
}t          t          |                     nd}|||dS )zDConvert an UpdateCollectionConfiguration to a JSON-serializable dictr2   r3   r4   Nr\   r]   r@   rA   r^   r_   rH   )rI   r   r   rP   rJ   r   re   validate_configrf   rF   r   r@   )rG   rQ   rR   rT   rV   rS   s         r$   r   r   )  s    **V$$K::g&&L	(	)	)B|3
		>6DDKK 	> 	> 	><<<===	> 	? 8,GGLL 	? 	? 	?=!==>>>	? (,I	~<<>> 		2*IIr}}///		--// I
 (R1111	 '  s0   A! !
B+A==BB 
B?(B::B?json_strc                 H    t          j        |           }t          |          S r8   )r9   r:   .load_update_collection_configuration_from_json)r   ri   s     r$   2load_update_collection_configuration_from_json_strr   U  s!     z(##H9(CCCr#   c                 J   |                      d          $|                      d          t          d          t                      }|                      d          t          | d                   |d<   |                      d          t	          | d                   |d<   |                      d          l| d         d         dk    rt          j        dt          d	
           n=t          | d         d                  }|	                    | d         d                   |d<   |S )z7Convert a JSON dict to an UpdateCollectionConfigurationr2   Nr3   r?   r4   r@   rA   rB   rC   rD   rF   rG   )
rI   rJ   r   r   r   rK   rL   rM   r   rO   )ri   r   rT   s      r$   r   r   ]  s,    ||F'HLL,A,A,MABBB*,,F ||F':8F;KLLv||G(<Xg=NOOw ||())5()&1X==M2"     +84H+I&+QRB+-+?+?-.x8, ,F'( Mr#   existing_hnsw_configupdate_hnsw_configc                 z    t          |           }g d}|D ]}||v r||         ||<   t          t          |          S )z6Overwrite a HNSWConfiguration with a new configuration)r   r   r   r   r   )rb   r   r   )r   r   r   update_fieldsfields        r$   overwrite_hnsw_configurationr   ~  s`    
 &''F  M  6 6&&&.u5F5M!6***r#   existing_spann_configupdate_spann_configc                 z    t          |           }ddg}|D ]}||v r||         ||<   t          t          |          S )z7Overwrite a SpannConfiguration with a new configurationr)   r   )rb   r   r(   )r   r   r   r   r   s        r$   overwrite_spann_configurationr     s\    
 '((FM
  7 7'''/6F5M"F+++r#   existing_embedding_functionupdate_embedding_functionc                    |                                  s|                                 rt          j        dt          d           | S |                                 |                                k    r:t          d|                                  d|                                 d          |                    |                                 |                                           |S )z7Overwrite an EmbeddingFunction with a new configurationz.cannot update legacy embedding function configrC   rD   z6Cannot update embedding function: incompatible types ( vs ))re   rK   rL   rM   rF   rJ   validate_config_updaterf   )r   r   s     r$   overwrite_embedding_functionr     s    #,,.. +2K2U2U2W2W +<	
 	
 	
 	

 +* #''))-F-K-K-M-MMM\+0022\ \8Q8V8V8X8X\ \ \
 
 	
 44#..002K2V2V2X2X   %$r#   existing_configupdate_configc                    |                     d          }|                     d          }||t          d          |                      d          }||t          ||          }|                      d          }||t          ||          }|                      d          }|                     d          }||t	          ||          }n|}t          |||          S )z<Overwrite a CollectionConfiguration with a new configurationr3   r2   Nr?   r4   rH   )rI   rJ   r   r   r   r1   )r   r   update_spannupdate_hnswupdated_hnsw_configupdated_spann_configupdated_embedding_function	update_efs           r$   "overwrite_collection_configurationr     s'   
 !$$W--L##F++KK$;ABBB *--f55&;+B:
 

 +..w77'L,D< , 
  

 "1!4!45I!J!J!!"677I%1)E*I* *&& *3&" "5   r#   r4   configuration_efc                    | }|{|                                  dk    rc|                                  |                                 k    r9t          d|                                   d|                                            dS )ap  
    Validates that there are no conflicting embedding functions between function parameter
    and collection configuration.

    Args:
        embedding_function: The embedding function provided as a parameter
        configuration_ef: The embedding function from collection configuration

    Returns:
        bool: True if there is a conflict, False otherwise
    Ndefaultz]Multiple embedding functions provided. Please provide only one. Embedding function conflict: r   )rF   rJ   )r4   r   s     r$   .validate_embedding_function_conflict_on_creater     s    $ %*:*F##%%22"''))-=-B-B-D-DDD i  qC  qH  qH  qJ  qJ  i  i  P`  Pe  Pe  Pg  Pg  i  i   4r#   persisted_ef_configc                 2   || |                                  dk    rz|                    d          e|                    d          |                                  k    r:t          d|                                   d|                    d                     dS )z
    Validates that there are no conflicting embedding functions between function parameter
    and collection configuration.
    Nr   rF   zAn embedding function already exists in the collection configuration, and a new one is provided. If this is intentional, please embed documents separately. Embedding function conflict: new: z vs persisted: )rF   rI   rJ   )r4   r   s     r$   +validate_embedding_function_conflict_on_getr     s     &+=+I##%%22#''//;#''//3E3J3J3L3LLL ]  Rd  Ri  Ri  Rk  Rk  ]  ]  |O  |S  |S  TZ  |[  |[  ]  ]   4r#   schemar   configurationc                 \   | j         j        | j         j        j        t          d          d}|| j        vrt          d| d          | j        |         }|j        |j        j        t          d| d          | j         j        j        |j        j        fD ]}d|v r|d         |j        j        t          d          |j        j        }|d         }d	|v r|d	         |_        d
|v r|d
         |_        d|v r|d         |_	        d|v r|d         |_
        d|v r|d         |_        n]d|v rY|d         Q|j        j        t          d          |j        j        }|d         }d|v r|d         |_        d	|v r|d	         |_        d|v r|d         |d         |j        _        | S )a  
    Updates a schema with configuration changes.
    Only updates fields that are present in the configuration update.

    Args:
        schema: The existing Schema object
        configuration: The configuration updates to apply

    Returns:
        Updated Schema object
    Nz2Schema is missing defaults.float_list.vector_indexz
#embeddingzSchema is missing keys[]z].float_list.vector_indexr2   z1Trying to update HNSW config but schema has SPANNr   r   r   r   r   r3   z1Trying to update SPANN config but schema has HNSWr)   r4   )defaults
float_listvector_indexrJ   keysrG   r2   r   r   r   r   r   r3   r)   r4   )	r   r   embedding_keyembedding_value_typesr   rQ   r   rR   r   s	            r$   +update_schema_from_collection_configurationr   !  s-   " 	"*?%2:MNNN MFK''C=CCCDDD"K6(0 +8@NmNNN
 
 	
 	"/(5 +Y +Y ]""}V'<'H"'/ !TUUU&-2K'/K k))(3K(@%++*5m*D'{**)4\)B&;..-89I-J*+--,7,H)%%-*@*L"(0 !TUUU'.4L(1L ,..-9/-J*l**)5k)B& !M1123?5BCW5XL2Mr#   r8   )r   r   r   r   r6   r   )9typingr   r   r   r   r   r   r9   chromadb.api.typesr	   r
   r   r   "chromadb.utils.embedding_functionsr   r   multiprocessingr   rK   r   r   r(   r1   strr=   r;   rZ   rY   rh   ro   rq   rs   ru   r{   rz   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r"   r#   r$   <module>r      s   A A A A A A A A A A A A A A A A                    & % % % % %  % % % % % %    	    	 	 	 	 	% 	 	 	 	4 4 4 4 4it 4 4 4 4DDD D D D7#s(^77 7 7 7t@1H @S @ @ @ @;-D ;cSVh ; ; ; ;|    iu    38n   6	 	 	 	 	y 	 	 	 	38n   .4 4 4 4 4IU 4 4 4 4O O"O O O O;38n;"; ; ; ;238n"   B .2Q Q)Q)*Q 	Q Q Q Q .2g g)g)*g 
#s(^g g g gV HL #)12C)D   .    iu    38n   "    y    
38n   4 4 4 4 4IU 4 4 4 4; ;"; ; ; ;&;;"; ; ; ;&!)!! ! ! !)))	#s(^) ) ) )XDD"D D D D38n"   B+++AX++ + + +*,-,1, , , , ,&%!2%0% % % % %8(,(0( ( ( ( (V !2301 
   @ !23!$sCx.1 
   (QQ%DQQ Q Q Q Q Qr#   