Zi1 UddlZddlZddlZddlZddlmZddlmZddlm Z ddl m Z m Z ddl m Z ddlmZddlmZdd lmZmZmZmZmZdd lmZmZddlZddlZdd lmZmZm Z dd l!m"Z"d dl#m$Z$d dl%m&Z&m'Z'd dl(m)Z)m*Z*m+Z+m,Z,d dl-m.Z.m/Z/d dl0m1Z1m2Z2m3Z3d dl4m5Z5m6Z6m7Z7m8Z8d dl9m:Z:ej;dej<Z=e GddZ>e?ej@ZAeeBd<GddeAZCGddej@eCZDGddejEjFZGGd d!ejEjFZHd"eId#eIfd$ZJd%eId#eIfd&ZKd'eId(eeLd#efd)ZMd*eDd'eId+eId,eNd-eIf d.ZOd/ZPeDZQdS)0N)deque) ExitStack)deepcopy) dataclassfield)datetime)chain)Path)AnyIteratorNoReturnOptionalUnion)quoteunquote)_DEFAULT_CALLBACK NoOpCallback TqdmCallback) isfilelike) constants)CommitOperationCopyCommitOperationDelete)EntryNotFoundErrorHfHubHTTPErrorRepositoryNotFoundErrorRevisionNotFoundError) hf_hub_urlhttp_get)HfApiLastCommitInfoRepoFile)HFValidationErrorhf_raise_for_status http_backoffhttp_stream_backoff)md5zy (^refs\/convert\/\w+) # `refs/convert/parquet` revisions | (^refs\/pr\/\d+) # PR revisions cxeZdZUdZeed<eed<eed<eed<eddZeeed <d efd Z dS) HfFileSystemResolvedPathzUData structure containing information about a resolved Hugging Face file system path. repo_typerepo_idrevision path_in_repoNF)defaultrepr _raw_revisionreturnctj|jd|jz}|jr'|d|jd|jdS|jtj kr4|dt|jd|jdS|d|jdS)N@/) rREPO_TYPES_URL_PREFIXESgetr*r+r0r-rstripr,DEFAULT_REVISION safe_revision)self repo_paths Y/root/projects/butler/venv/lib/python3.11/site-packages/huggingface_hub/hf_file_system.py unresolvez"HfFileSystemResolvedPath.unresolve4s599$."MMPTP\\   BJJ$"4JJt7HJJQQRUVV V ]i8 8 8TT- ">">TTARTT[[\_`` `55$"355<r?r=r)r)(s__NNN LLLMMM$)5E#B#B#BM8C=BBBB3BBBBBBr?r) _cached_basec(eZdZdZfdZdZxZS)_Cacheda Metaclass for caching HfFileSystem instances according to the args. This creates an additional reference to the filesystem, which prevents the filesystem from being garbage collected when all *user* references go away. A call to the :meth:`AbstractFileSystem.clear_instance_cache` must *also* be made for a filesystem instance to be garbage collected. This is a slightly modified version of `fsspec.spec._Cached` to improve it. In particular in `_tokenize` the pid isn't taken into account for the `fs_token` used to identify cached instances. The `fs_token` logic is also robust to defaults values and the order of the args. Finally new instances reuse the states from sister instances in the main thread. cHtj|i|i|_dSN)super__init___cache)clsargskwargs __class__s r=rMz_Cached.__init__Rs,$)&))) r?cT|dd}|j|tjg|Ri|}|j|tjjg|Ri|}|s$|jr||jvr||_|j|Stj |g|Ri|}|sZ|jrS||jvrJ|j| }| D]\}} t||| ||_||_||_|jr|s||_||j|<|S)Nskip_instance_cacheF)pop _tokenize threading get_ident main_threadidentcachablerN_latesttype__call___get_instance_stateitemssetattr _fs_token_ storage_argsstorage_options) rOrPrQskipfs_tokenfs_token_main_threadobjinstance_stateattr state_values r=r^z_Cached.__call__Zsrzz/77 3=i&9&;&;MdMMMfMM,s}S)2G2I2I2OaRVaaaZ`aa   SZ)?)?"CK:h' '-5d555f55C 4CL 4-ASZ-O-O!$,@!A!U!U!W!W)7)=)=)?)?44%D+C{3333%CN#C "(C | +D +& '* 8$Jr?)r@rArBrCrMr^ __classcell__rRs@r=rIrIBsQ  r?rIc4eZdZdZdZdZddddddeedee edfdee d ee ffd Z e d e d efd Z dededeed ee eeffdZd5dedeed efdZd5deed dfdZ d6dededee deed edf dZd5dedeed dfdZ d7dede dee deed df dZ d8dede d e deed eeeeeefff d!Z d9dede d e deed ee dee f d"Zded eeeeeeefffd# Zd5dedee d eeffd$ Z d:dedee d%e de d e deed eeeeeeeeffffd&Zd5d'ed(edeed dfd)Z ded e!fd*Z"d;ded e deed eeeffd+Z#d,Z$d-Z%d.Z&ded efd/Z'e(dfd<fd0 Z)e*d1Z+d2Z,d3Z-d4Z.xZ/S)= HfFileSystemaL Access a remote Hugging Face Hub repository as if were a local file system. > [!WARNING] > [`HfFileSystem`] provides fsspec compatibility, which is useful for libraries that require it (e.g., reading > Hugging Face datasets directly with `pandas`). However, it introduces additional overhead due to this compatibility > layer. For better performance and reliability, it's recommended to use `HfApi` methods when possible. Args: endpoint (`str`, *optional*): Endpoint of the Hub. Defaults to . token (`bool` or `str`, *optional*): A valid user access token (string). Defaults to the locally saved token, which is the recommended method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). To disable authentication, pass `False`. block_size (`int`, *optional*): Block size for reading and writing files. expand_info (`bool`, *optional*): Whether to expand the information of the files. **storage_options (`dict`, *optional*): Additional options for the filesystem. See [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.__init__). Usage: ```python >>> from huggingface_hub import hffs >>> # List files >>> hffs.glob("my-username/my-model/*.bin") ['my-username/my-model/pytorch_model.bin'] >>> hffs.ls("datasets/my-username/my-dataset", detail=False) ['datasets/my-username/my-dataset/.gitattributes', 'datasets/my-username/my-dataset/README.md', 'datasets/my-username/my-dataset/data.json'] >>> # Read/write files >>> with hffs.open("my-username/my-model/pytorch_model.bin") as f: ... data = f.read() >>> with hffs.open("my-username/my-model/pytorch_model.bin", "wb") as f: ... f.write(data) ``` Specify a token for authentication: ```python >>> from huggingface_hub import HfFileSystem >>> hffs = HfFileSystem(token=token) ``` r3hfN)endpointtoken block_size expand_inforqrrrsrtctj|i||p tj|_||_t |||_||_||_ i|_ i|_ dS)N)rqrr) rLrMrENDPOINTrqrrr _apirsrt_repo_and_revision_exists_cachedircache)r;rqrrrsrtrPrdrRs r=rMzHfFileSystem.__init__st $2/222 6I$6  85999 $&  ,:< r?threading_identr1cBdp tjd<dd<fdtD|||f}t t |}|S)zDeterministic token for cachingrqrrc"i|] }|| SrFrF).0keyrQs r= z*HfFileSystem._tokenize..s===s#vc{===r?)r7rrvsortedr'rDencode hexdigest)rOrzrPrQ tokenize_argshs ` r=rVzHfFileSystem._tokenizes$ZZ 33Iy7Iz **W--w====fVnn===otV< M""))++ , ,{{}}r?r*r+r,c|||f|jvr |j|||tjd|j|||f<d|j||df<nj#t t f$r(}d|f|j|||f<d|f|j||df<Yd}~n6d}~wt$r&}d|f|j|||f<d|j||df<Yd}~nd}~wwxYw|j|||fS)N)r,r*timeout)TNF)rxrw repo_inforHF_HUB_ETAG_TIMEOUTrr#r)r;r*r+r,es r=_repo_and_revision_existz%HfFileSystem._repo_and_revision_existsT w )1U U U ^ ##h)YMj$Xb4i(5STS]4i$5OPP,->? \ \ \W\^_W_4i(5STSXZ[S[4i$5OPPPPPP( ^ ^ ^W\^_W_4i(5STS]4i$5OPPPPPP ^ 3Y4RSSs#(AB8"B B8B33B8pathcdttdttdttfd}||}|std|dddzt jvr?d|vrtd|dd\}}t j|}n t j }| ddkrTd d |dd d vr|d d\}}d|vrt |}|[|d |fvrCtd |d}|}n|dd\}}nd }|t#||}||||\} } | st'|| nd }d |dd d } d |dd d } |dd} d |ddd }| }| }||||\} } | s]t)| t*t,fr1| }|}||||\} }| st'|| nyt'|| nh|}d }d |vr3|d d\}}|t#||}nd }||||\} }| std||n t j}t1||||| S)a Resolve a Hugging Face file system path into its components. Args: path (`str`): Path to resolve. revision (`str`, *optional*): The revision of the repo to resolve. Defaults to the revision specified in the path. Returns: [`HfFileSystemResolvedPath`]: Resolved path information containing `repo_type`, `repo_id`, `revision` and `path_in_repo`. Raises: `ValueError`: If path contains conflicting revision information. `NotImplementedError`: If trying to list repositories. revision_in_pathr,r1cL||||krtd|d|dn|}|S)NzRevision specified in path ("z ") and in `revision` argument ("z") are not the same.) ValueError)rr,s r=%_align_revision_in_path_with_revisionzHHfFileSystem.resolve_path.._align_revision_in_path_with_revisions^##/4D4P4P$-8H--jr--- ,Or?z0Access to repositories lists is not implemented.r5rrr4Nr3)r0)rrD_strip_protocolNotImplementedErrorsplitrr6valuesREPO_TYPES_MAPPINGREPO_TYPE_MODELcountjoinSPECIAL_REFS_REVISION_REGEXsearchgroupsublstriprr_raise_file_not_found isinstancerr#r9r))r;rr,rr*r+rmatchr-repo_and_revision_existerrrepo_id_with_namespacepath_in_repo_with_namespacerepo_id_without_namespacepath_in_repo_without_namespace_s r= resolve_pathzHfFileSystem.resolve_paths( &sm 7?}  c]    ##D)) 2%&XYY Y ZZ__Q # %)J)Q)Q)S)S S S$)*\]]]"jja00OIt!4Y?II!1I ::c??Q  chhtzz#rr23333,0JJsA,>,>))***7>>?OPPE(X$ 9N-N-N'B'F'FrK['\'\'c'cdg'h'h +0;;==((9I9O9OPSUV9W9W6(,,#%L@@IYAZAZ\dee/3/L/LYX_ai/j/j,'.5)$444#' ),$**S//"1"2E)F)F&.1hhtzz#qrr7J.K.K+,0JJsOOA,>)14$**S//!"":M1N1N.0: /3/L/LYX_ai/j/j,'.9!#(?AR'STT9";'E 595R5RS\^ego5p5p2/6=1$<<<-dC888GLd{{,0JJsA,>,>))@@IYAZAZ\dee#' )-)F)FyRY[c)d)d & #Q* ^)*\]]]'3889S' 7Hlbrssssr?c|s4|j|jdS||}|}|r2|j|d||}|2|jsW|j|j|j dfd|j|j|j |j fddSdS)ac Clear the cache for a given path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.invalidate_cache). Args: path (`str`, *optional*): Path to clear from cache. If not provided, clear the entire cache. N) ryclearrxrr>rU_parentr-r*r+r,)r;r resolved_paths r=invalidate_cachezHfFileSystem.invalidate_cache>s  M   ! ! !  0 6 6 8 8 8 8 8 --d33M **,,D * !!$---||D)) * !- 488-:QS`Shjn9oquvvv488",m.C]E[\^b  r?rbmode)HfFileSystemFileHfFileSystemStreamFilec ||n|j}|||d<d|vrtd|dkrt||f||d|St||f||d|S)Nrsaz/Appending to remote files is not yet supported.r)rr,)rsrrr)r;rrrsr,rQs r=_openzHfFileSystem._openZs$.#9ZZt  !#-F< $;;%&WXX X ??)$]4(]]V\]] ]#D$WTHWWPVWW Wr?c D|||}|j|j|j|j|j|j|d|d| | dS)Nr,commit_messagecommit_description)r-r+rrr*r,rrr) rrw delete_filer-r+rrr*r,r7rr>)r;rr,rQrs r=_rmzHfFileSystem._rmls))$)BB  &3!)*#-"+!::&677%zz*>??     =#:#:#<#<=====r?F recursivemaxdepthc ||}||||}fd|D}d|D} d|d} | |rdndz } | |d |dndz } j|j|jj| |j|d | |d   | d S)a Delete files from a repository. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.rm). > [!WARNING] > Note: When possible, use `HfApi.delete_file()` for better performance. Args: path (`str`): Path to delete. recursive (`bool`, *optional*): If True, delete directory and all its contents. Defaults to False. maxdepth (`int`, *optional*): Maximum number of subdirectories to visit when deleting recursively. revision (`str`, *optional*): The git revision to delete from. r)rrr,cng|]1}||j2SrF)isdirrr-)r}rr;s r= z#HfFileSystem.rm..s?hhh$W[WaWabfWgWgh**400=hhhr?c.g|]}t|S))r-)r)r}r-s r=rz#HfFileSystem.rm..s$iii<+FFFiiir?zDelete  z recursively r3Nz up to depth rr)r+r*rr operationsr,rrr) r expand_pathrw create_commitr+r*rrr,r7rr>) r;rrrr,rQrpaths paths_in_reporrs ` r=rmzHfFileSystem.rmys16))$)BB   XX` aahhhh%hhh ii[hiii *4***I=..2=8L44444RTT !)#-*!"+!::&6GG%zz*>??     =#:#:#<#<=====r?Tdetailrefreshc ||}| |jf||d|}n|#t$ro|jst d|j|f||d|}fd|D}t|dkrt dYnwxYw|r|n d|DS)a List the contents of a directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.ls). > [!WARNING] > Note: When possible, use `HfApi.list_repo_tree()` for better performance. Args: path (`str`): Path to the directory. detail (`bool`, *optional*): If True, returns a list of dictionaries containing file information. If False, returns a list of file paths. Defaults to True. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `list[Union[str, dict[str, Any]]]`: List of file paths (if detail=False) or list of file information dictionaries (if detail=True). r)rr,Nc,g|]}|dk|SnamerFr}ors r=rz#HfFileSystem.ls..s'777QvY$%6%61%6%6%6r?rcg|] }|d SrrFr}rs r=rz#HfFileSystem.ls..s":":":1V9":":":r?)rr>_ls_treerr-rrlen)r;rrrr,rQrouts ` r=lszHfFileSystem.lss4))$)BB &&(( 2$-SgSSFSSCC! 2 2 2 - 2%dD111$- T 2 2aGhaaZ`aaC7777c777C3xx1}}%dD111 2:ss":":c":":"::sAA6B98B9c ||njjnd}||}|}t|j|j|jd|j}g} |jvrF|sCj|} | | g} |rdtfd| D} | r| \} ||krt| djvr| | dnIj| d} | | | fd| D| g}|rfd | D}|r| s|rN|rKtj| |z}|d s||ks|t#| |vr|d n||/t)|dd }||z}fd | D} t-jD]5}|d zrj|d6jd| |d ||| n'j|j|j|||j|j}|D]}|d z|jz}t;|t<r*||jd|j |j!|j"|j#|j$d}n|dd|j%|j#d}|d}j&|g ||t)|dd ||kr| || S)NFrr3r-r0rc0g|]}|ddk|fSr] directoryrFr} path_infodepths r=rz)HfFileSystem._ls_tree..s/oooIiX^N_cnNnNneY'NnNnNnr?rc6g|]}|ddkdz|fS)r]rrrFrs r=rz)HfFileSystem._ls_tree..s<!"!"!"(1'0'8K'G'G&+QY $:'G'G'Gr?cVg|]%}|d |d&S) last_commitNr)r)r}rr;s r=rz)HfFileSystem._ls_tree..s3$f$f$fQ}M]MeT\\!F)%<%.s6UUUQQvY-A-A+PSBS-T-TUqUUUr?T)rrr,rtr)rexpandr,r*filersizer]blob_idlfsxet_hashrsecurityrrrrr]tree_idr)'rtrr>r)r*r+r,r0ryextendrpopleftappendosr commonprefixendswithr r8rrrlistrrUrrwlist_repo_treer-rr"rrrrrrr setdefault)r;rrrr,rtrr root_pathrcached_path_infosdirs_not_in_dircache dirs_to_visitdir_infodirs_not_expanded common_prefixcommon_path_depth cached_pathtreer cache_pathcache_path_info parent_pathrrs` @@r=rzHfFileSystem._ls_trees'2KKTM]Mi9I9Iot ))$)BB &&((,  #  !  "'5    )++  4=  $ d 3  JJ( ) ) )#%   %oooo9Jooo!! $&3&;&;&=&=OE8'5H+<+<#F+4=@@0778HIIII04 hv>N0O-JJ'8999)00!"!"!"!"5F!"!"!"$ !#  g$f$f$f$fc$f$f$f! 2   HY !# 4 45IL]5] ^ ^ %--c225$ 11$.BDU(V(VVV"((---m44 '(3CIIKK(@(F(Fs(K(K% 11HUUUU#UUU#' #6#6==K"--kC.?@@= ))+t<<< !!+t444 MM#"+ $!)$/!) "   9++%*#"&/'1 ,D" 0 0 &_y~= i22 * ) &#,#4(}$-$6'0'<$-$6 ' 'OO!+ ! +#,#4'0'< ''O#ll?6+BCC  ((b99@@QQQ"3t99;;/55c::#u'8'8JJ/// r?c/K|||d}tj|g|Ri|Ed{VdS)a Return all files below the given path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.walk). Args: path (`str`): Root path to list files from. Returns: `Iterator[tuple[str, list[str], list[str]]]`: An iterator of (path, list of directory names, list of file names) tuples. r,rN)rr7r>rLwalk)r;rrPrQrRs r=rzHfFileSystem.walkIsw   :0F0F GGQQSS577<6t666v66666666666r?c |||d}tj|fd|i|S)ah Find files by glob-matching. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.glob). Args: path (`str`): Path pattern to match. Returns: `list[str]`: List of paths matching the pattern. r,rr)rr7r>rLglob)r;rrrQrRs r=r zHfFileSystem.globYsX   :0F0F GGQQSSuww|D>>8>v>>>r?withdirsc  ||dkrtd|||}|} |j|fd||j|d| |s d D n)|j|fd|ji|} | d d kr| g zn d D nE#t $r8 |j|fd|i|d d kr|ii ni n#t$ri YnwxYwYnwxYwt } |s| S fd | DS)a List all files below path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.find). Args: path (`str`): Root path to list files from. maxdepth (`int`, *optional*): Maximum depth to descend into subdirectories. withdirs (`bool`, *optional*): Include directory paths in the output. Defaults to False. detail (`bool`, *optional*): If True, returns a dict mapping paths to file information. Defaults to False. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `Union[list[str], dict[str, dict[str, Any]]]`: List of paths or dict of file information. Nrzmaxdepth must be at least 1rT)rrr,rc*g|]}|ddk|SrrFrs r=rz%HfFileSystem.find..s&BBBQ6k)A)Aq)A)A)Ar?r,r]rc i|] }|d| SrrFrs r=rz%HfFileSystem.find..s---A1V9a---r?rc"i|] }|| SrFrF)r}rrs r=rz%HfFileSystem.find..s666D#d)666r?) rrr>rr,inforFileNotFoundErrorr) r;rrr rrr,rQrrnamesrs @r=findzHfFileSystem.findis@  HqLL:;; ;))$)BB &&(( .$- $g @VaimsC UBB#BBB&DIdVV]5KVvVV +4V+< +K+KykC''QT-----CC!"    49T??H???G6QQ*CCC$     "s  7L6666666 6s6B"" C$-!CC$ CC$CC$#C$path1path2c |||}|||}|j|jko|j|jk}|rd|d|}|j|j|j|j|d||ddt|j|j|jgn| |d |j5} | } d d d n #1swxYwYd|d|}|j | |j|j|j |j|j|d||d | | | | d S) a Copy a file within or between repositories. > [!WARNING] > Note: When possible, use `HfApi.upload_file()` for better performance. Args: path1 (`str`): Source path to copy from. path2 (`str`): Destination path to copy to. revision (`str`, *optional*): The git revision to copy from. rzCopy z to rrr3)src_path_in_repor- src_revision)r+r*r,rrrrNpath_or_fileobjr-r+rrr*r,rrr)rr*r+rwrr,r7rr-openread upload_filerrrr>) r;rrr,rQresolved_path1resolved_path2 same_reporfcontents r=cp_filezHfFileSystem.cp_files< **58*DD**58*DD  $(@ @ u^E[_m_uEu   7U7777N I # #&.(2'0%zz*:NKK#)::.BB#G#G')7)D%3%@%3%< $    5$1HII #Q&&(( # # # # # # # # # # # # # # #7U7777N I ! ! '+8&.j(2'0%zz*:NKK#)::.B#C#C "    >#;#;#=#=>>> >#;#;#=#=>>>>>s.DDDc F|j|fii|ddi}|ddS)ai Get the last modified time of a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.modified). Args: path (`str`): Path to the file. Returns: `datetime`: Last commit date of the file. rtTrdater)r;rrQrs r=modifiedzHfFileSystem.modifieds=tyAA!@F!@M4!@!@AAM"6**r?c  ||}||d|j|jnd}|jsedddd |rZ|j|j|j|j d }i dt|j |j |j d  nd |}|s||jvr||||jvr3fd |j|D} | st#d| d |s |r r d |j|j|j||j |j} | st#d| d} t'|j|j|j d|j} t+| t,r5| dz| jz| jd| j| j| j| j| jd n| dz| jzdd| j| jd |s fddD J S)a Get information about a file or directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.info). > [!WARNING] > Note: When possible, use `HfApi.get_paths_info()` or `HfApi.repo_info()` for better performance. Args: path (`str`): Path to get info for. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to get info from. Returns: `dict[str, Any]`: Dictionary containing file information (type, size, commit info, etc.). rrtNFrr)rrr]r)r*r,)oidtitler%)rrc,g|]}|dk|SrrFrs r=rz%HfFileSystem.info..%s'SSSa6dARARARARARr?r)rr,r*r3rr5rrrc"i|] }|| SrFrF)r}krs r=rz%HfFileSystem.info..OsGGG1c!fGGGr?)rrr])rr>r7rtr-rwlist_repo_commitsr+r*r,r! commit_idr+ created_atrryrrget_paths_infor)r0rr"rrrrrrrr)r;rrr,rQrrtrrout1 paths_inforrrs ` @r=rzHfFileSystem.infos*))$)BB &&((jj t/?/K4++QV  )E H## C  "i99!)]5LWdWm: ##1'19JQ\Qg$$$C,,t,,K %;dm#C#C $$$dm++SSSS4=#=SSS6)$5551g& H#+++#+#mBTB\!Y55!)!.&*3+5 6 "6)$555&qM 4!+!)!*!#"/"=  )++ i22 )C). @ ) &#,#4(}$-$6'0'<$-$6  CC!*C). @ ! +#,#4'0'< C#HGGGG.FGGGC r?c |ddr|||j|fi|dS#YdSxYw)a Check if a file exists. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.exists). > [!WARNING] > Note: When possible, use `HfApi.file_exists()` for better performance. Args: path (`str`): Path to check. Returns: `bool`: True if file exists, False otherwise. rFT)r7rr)r;rrQs r=existszHfFileSystem.existsSs`  zz)U++ ,%%d+++ DId % %f % % %4 55s 9=Acd ||ddkS#t$rYdSwxYw)ai Check if a path is a directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isdir). Args: path (`str`): Path to check. Returns: `bool`: True if path is a directory, False otherwise. r]rF)rOSErrorr;rs r=rzHfFileSystem.isdirlsC 99T??6*k9 9   55 s ! //cR ||ddkS#YdSxYw)a` Check if a path is a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isfile). Args: path (`str`): Path to check. Returns: `bool`: True if path is a file, False otherwise. r]rFr&r9s r=isfilezHfFileSystem.isfile~s1 99T??6*f4 4 55s!&c||}t|j|j|j|j|j}||r|ddd}|S)z Get the HTTP URL of the given path. Args: path (`str`): Path to get URL for. Returns: `str`: HTTP URL to access the file or directory on the Hub. )r*r,rqz /resolve/z/tree/r) rrr+r-r*r,rqrreplace)r;rrurls r=r>zHfFileSystem.urlsx))$//   !  &#-"+]     ::d   8++k8Q77C r?c |d}t|dhz }t|tt frt |dkrtj||f||d|St|r|}n-| |rtj |ddSt|ttfr3tj tj|dd}|t#|d}d}|} ||| } ||| d } ||  t-t/| j| j| j| j|j ||| d|jt|t r|jnd | | |r|!dSdS#|r|!wwxYw) aC Copy single remote file to local. > [!WARNING] > Note: When possible, use `HfApi.hf_hub_download()` for better performance. Args: rpath (`str`): Remote path to download from. lpath (`str`): Local path to download to. callback (`Callback`, *optional*): Optional callback to track download progress. Defaults to no callback. outfile (`IO`, *optional*): Optional file-like object to write to. If provided, `lpath` is ignored. r,r)callbackoutfileT)exist_okNFwbrrr+r,filenamer*rq)r> temp_filedisplayed_filename expected_size resume_sizeheaders _tqdm_bar)"r7setkeysrrrrrLget_filerrrmakedirsrDr rdirnamertellrrset_sizerrr+r,r-r*rqrw_build_hf_headerstqdmseekclose) r;rpathlpathr@rArQr,unhandled_kwargs close_file initial_posresolve_remote_pathrHrRs r=rNzHfFileSystem.get_files[$::j))v{{}}-- <(\<$@AA `SIYEZEZ]^E^E^$577#E5_8W__X^__ _ e   GG ZZ    K - - - -4 ec4[ ) ) ? K.. > > > > ?5$''GJllnn #///II %( ;;FC -((( /7090=1;!] "#(+ 3355+5h +M+MW(--SW     LL % % %    z   s B H..Ic td)zA context within which files are committed together upon exit Requires the file class to implement `.commit()` and `.discard()` for the normal and exception cases. (Transactional commits are not supported.rr;s r= transactionzHfFileSystem.transactions""LMMMr?c td)z@Begin write transaction for deferring files, non-context versionr^r_r`s r=start_transactionzHfFileSystem.start_transactions""LMMMr?cntt||j|j|ffSrK) make_instancer]rcrdr_r`s r= __reduce__zHfFileSystem.__reduce__s8 JJ     $ $ & &    r?cTt|jt|jdS)N)ryrx)rryrxr`s r=r_z HfFileSystem._get_instance_states, ///78\/]/]   r?rK)rNN)FNN)TFN)FFNNN)NFFFN)FNr1N)0r@rArBrC root_markerprotocolrrDrboolintrM classmethodrVtuple Exceptionrr)rrrrrrdictr rrr rr rr#rr'rr6rr;r>rrNpropertyrarcrfr_rlrms@r=rorots..`KH #'(,$(&* <<<3-<T3_% < SM < d^ <<<<<<0     [ TT'*T6>smT tXi(( )TTTT&]t]t]t ]tIa]t]t]t]t~Xc]d>$("& XXXXSM X 3- X ; <XXXX$ > > >x} >$ > > > >  "&"& ,>,>,>,>3- ,> 3- ,> ,>,>,>,>^`d&;&;&;!%&;7;&;OWX[}&; eCc3h'( )&;&;&;&;V "&&*"&xxxx x 3- x d^ x3-xxxxt77(5d3iQUVYQZAZ;[2\777777 ??? ?4PS9??????&#'"&=7=7=73-=7 =7  =7  =73-=7 tCy$sDcN233 4=7=7=7=7~6?6?S6?6? 6?[_6?6?6?6?p+S+x++++ aaatax}abfgjlogobpaaaaF2$$./@B B B B B B B HNNXNNNN           r?ro) metaclassceZdZddededeeffd ZfdZdeded e fd Z dd Z dd e d dfdZ dfd Zd efdZxZS)rNfsrr,c ( ||||_n>#t$r1}d|ddvrt|d|d}~wwxYwt j||jfi||dS)Nrwrr3B. Make sure the repository and revision exist before writing data.)rrrr7rLrMr>)r;rtrr,rQrrRs r=rMzHfFileSystemFile.__init__ s !#!I!ID      fjj,,,,']]]    T/99;;FFvFFF s A,AAcht|dsdStSNr)hasattrrL__del__r;rRs r=r{zHfFileSystemFile.__del__s.t_--  Fww   r?startendr1cJdd|d|dz i|jj}t|jj|jj|jj|jj|jj }td||tj }t||jS)Nrangebytes=-rrDGETrJr)rtrwrSrrr+r,r-r*rqr%rHF_HUB_DOWNLOAD_TIMEOUTr$r")r;r}r~rJr>rs r= _fetch_rangezHfFileSystemFile._fetch_ranges /e//cAg// gl,,.. &.'0'4(2W%     Wi>_ ` ` `Ayr?c<tjdd|_dS)Nzhffs-F)prefixdelete)tempfileNamedTemporaryFilerFr`s r=_initiate_uploadz!HfFileSystemFile._initiate_upload/s!4GERRRr?Ffinalc |jd|j}|j||r|j|jj|jj |j j |j j |jj |j j|j j|jd|jdt%j|jj |j|j dSdS)Nrrrrr)bufferrUrrFwriterVrtrwrrrr-r+rrr*r,rQr7rremoverr>)r;rblocks r= _upload_chunkzHfFileSystemFile._upload_chunk2s1    "" U###   N " " " GL $ $ $ 3!/<*2gm,6+4#{/?@@#';??3G#H#H %    Idn) * * * G $ $'1133 %       r?r)c\|jdkr||dkrx|jdkrm|j|jdd5}|}|xjt |z c_|cdddS#1swxYwYt|S)zRead remote file. If `length` is not provided or is -1, the entire file is downloaded and read. On POSIX systems the file is loaded in memory directly. Otherwise, the file is downloaded to a temporary file and read from there. rNr)r)rs)rlocrtrrrrrL)r;lengthr!rrRs r=rzHfFileSystemFile.readGs 9  &.FbLLdhRSmmdi!<< ffhhCHH$                ww||F###s3BB Bc@|j|jSrKrtr>rr`s r=r>zHfFileSystemFile.urlTw{{49%%%r?rKrh)Fr))r@rArBrorDrrMr{rlbytesrrrkrrr>rlrms@r=rr s  < s hsm      !!!!! #CE SSSS4D* $ $ $ $ $ $&S&&&&&&&&r?rc eZdZ ddedededeed ed ef fd Zdd ed efdZddefdZ dde e dede fdZ defdZ fdZdZdZxZS)rrNrnonertrrr,rs cache_typec p|dkrtd||dkrtd|d|vrtd|d ||||_nB#t$r5}d|d d vrt|d |Yd}~nd}~wwxYw|jdd |_tj||jf|||d |d|_ |t|_ d|_ t|_dS)Nrz:HfFileSystemStreamFile only supports block_size=0 but got rz?HfFileSystemStreamFile only supports cache_type='none' but got rvz;HfFileSystemStreamFile only supports reading but got mode=''rrr3rw)rrrrsr)rrrrr7r>detailsrLrMresponser _exit_stack_stream_iterator bytearray_stream_buffer) r;rtrrr,rsrrQrrRs r=rMzHfFileSystemStreamFile.__init__Ys ??fZdffgg g   k_ikkll l $;;b[_bbbcc c !#!I!ID      fjj,,,,']]]-,,,,  !% 2 < < > >MM  ",,.. 59j]g  kq   37  $;;;?'kks A'' B&1+B!!B&rwhencecb|dkr|dkrdS||jkr|dkrdStd)NrrzCannot seek streaming HF file)rr)r;rrs r=rUzHfFileSystemStreamFile.seek|s? !88! F $(??v{{ F8999r?r)rch|j|d} |j|jdS||j|}|xjt |z c_|S#t $r<|j|j|r|d}YnwxYw)zRead the remote file. If the file is already open, we reuse the connection. Otherwise, open a new connection and read from it. If reading the stream fails, we retry with a new connection. NFTr?)r_open_connectionr_read_from_streamrrrorV)r;r retried_oncers r=rzHfFileSystemStreamFile.reads =  ! ! # # #  $ $=(D,A,I3,,T-BFKKCHH$  $ $ $=,M'')))%%'''#  $ $sA*9A**AB0/B0iteratorr1c|dkrdS|dkrVt|j}|j|D]}||t |S|t |jkr(t |jd|}|jd|=|St|j}|j|D]|}|t |z }|t |kr||=||d||j||dt |S)zRead up to `length` bytes from stream buffer and stream. If length < 0, read until EOF. If EOF is reached before length, fewer bytes may be returned. rr?N)rrrrrr)r;rrbufchunkresultneeds r=rz(HfFileSystemStreamFile._read_from_streamse Q;;3 A::D/00C   % % ' ' '! " " 5!!!!::  S,-- - -4.ww788F#GVG,M+,, !!###  ECHH$Dc%jj   5!!!! 5$<(((#**5<888Szzr?c@|j|jSrKrr`s r=r>zHfFileSystemStreamFile.urlrr?ct|dsdS|jtSry)rzrrVrLr{r|s r=r{zHfFileSystemStreamFile.__del__sCt_--  F    ww   r?cZt|j|j|j|j|jjffSrK)reopenrtrr blocksizecacherr`s r=rfz!HfFileSystemStreamFile.__reduce__s$DIt~tzWWWr?c||jd|_t|jj|jj|jj|jj|j j }|j j }|j dkrd|j d|d<|jt!d||t"j|_ t)|jn0#t*$r#}|jjd kr d|_Yd}~dSd}~wwxYw|j|_dS) z%Open a connection to the remote file.NrDrrrRangerri)rrrrrr+r,r-r*rtrqrwrSrr enter_contextr&rrrr$r status_code iter_bytes)r;r>rJrs r=rz'HfFileSystemStreamFile._open_connectionsQ !!### $&.'0'4(2W%    ',0022 8a<<3333GG (66 !9          . . . .   z%,, $    !% 8 8 : :sC.. D8DDD)rNrr)rr)r@rArBrorDrrlrMrUrr rrr>r{rfrrlrms@r=rrXs_ "& !*!* !*!* !* 3- !*  !*!*!*!*!*!*!*F:::S::::$$3$$$$8  (5/ 3 PU    D&S&&&&!!!!!XXX";";";";";";";r?rr,r1cXt|r|nt|SrK)rr safe_quoters r=r:r:s(288BB \88 S[H\H\\r?sc$t|dS)Nr3)safe)r)rs r=rrs    r?rrc|}t|tr|d}n5t|tr|d}nt|tr|d}t ||)Nz (repository not found)z (revision not found)z (invalid repository id))rrrr#r)rrmsgs r=rrs~ C#.//0... C. / /0,,, C* + +0/// C c)r?rtrrsrc4|||||S)Nr)r)rtrrrsrs r=rrs 774dzj7 Q QQr?cl||i|}|D]\}}t||||SrK)r`ra)rOrPrQrirtrjrks r=rere sP d f  B+1133''kD+&&&& Ir?)RrrerrW collectionsr contextlibrcopyr dataclassesrrr itertoolsr pathlibr typingr r r rr urllib.parserrfsspechttpxfsspec.callbacksrrr fsspec.utilsrr3r _commit_apirrerrorsrrrr file_downloadrrhf_apir r!r"utilsr#r$r%r&utils.insecure_hashlibr'compileVERBOSErr)r]AbstractFileSystemrGrErIrospecAbstractBufferedFilerrrDr:rrorrlrrehffsrFr?r=rs  ((((((((;;;;;;;;;;;;;;'''''''' JJJJJJJJJJ######CCCCCCCCffffffffffff////////3333333333\\\\\\\\\\\\'''''')bj J  BBBBBBB B,D233 c333/////l///dU U U U U 6,U U U U pI&I&I&I&I&v{7I&I&I&XX;X;X;X;X;V[=X;X;X;v]C]C]]]]##**(9*=*(****R|R3RcRsRPSRRRR|~~r?