ZiXm4 dZddlZddlZddlZddlZddlmZddlmZddl m Z ddl m Z m Z mZmZmZmZmZmZddlZddlmZdd lmZmZmZmZmZmZmZmZdd l m!Z!m"Z"d d l#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*e*d Z+edGddZ,ej-dZ.hdZ/hdZ0hdZ1ej-dZ2dddddZ3iZ4e5e1D]Z6e6e4e6<e6de4e6d<e6de4e6d<e de4e7e8 Z9edGd!d"Z:eeeefZ;eeed#eeee ee7ejOd?fd@e eAejOdAfdBe eeKe7ejOdCdDdEfdFe e$ejOdGfdHe eAejOdIdJdKfdLe ee9ejOdMfdNe eePejOdOfd$dfdPZQe+MgdQ< dld9e eKe7ejRdRfd>e ee7ejOd?fdSe eAejOdTdUdVfdWe eAejOdXfd$df dYZSe+MdZd[g< dld>e ee7ejOd?fdSe eAejOdTdUdVfdWe eAejOdXfd$dfd\ZTe+Mgd]<e&jUdddd=d=dfd^e%d_e&d`e'd>e ee7ejOdafdbe ee7ejOdcfdde eAejOdedffdge eAejOdhdifdje(d$dfdkZVdS)mzHContains the 'hf cache' command group with cache management subcommands.N) defaultdict) dataclass)Enum) AnnotatedAnyCallableDictListMappingOptionalTuple)CLIError)ANSICachedRepoInfoCachedRevisionInfo CacheNotFound HFCacheInfo _format_sizescan_cache_dirtabulate)parse_duration parse_size) OutputFormat RepoIdArg RepoTypeOpt RevisionOptTokenOpt get_hf_api typer_factoryzManage local cache directory.)helpT)frozencfeZdZUeeed<eeeefed<e edfed<dS)_DeletionResolution revisionsselected.missingN) __name__ __module__ __qualname__ frozensetstr__annotations__dictrrtupleT/root/projects/butler/venv/lib/python3.11/site-packages/huggingface_hub/cli/cache.pyr%r%:sR~>9-?#@@AAAA 38_r2r%zA^(?P[a-zA-Z_]+)\s*(?P==|!=|>=|<=|>|<|=)\s*(?P.+)$><=>!=<=>=>refssizetypeaccessedmodified>namer;r=r>z.^(?P[a-zA-Z_]+)(?::(?Pasc|desc))?$descasc)r=r>r;r?z:asc_ascz:desc_desc SortOptions)r<modulec2eZdZUdZeed<eed<eed<dS)CacheDeletionCountsz>Simple counters summarizing cache deletions for CLI messaging. repo_countpartial_revision_counttotal_revision_countN)r)r*r+__doc__intr.r1r2r3rGrGYs9HHOOOr2rGselected_by_reporeturnc d}d}d}|D]S\}}|t|z }t|t|jkr|dz }|t|z }T||z }t|||S)z(Summarize deletions across repositories.rr)itemslenr&rG)rMrHtotal_revisionsrevisions_in_full_reposrepor&rIs r3summarize_deletionsrUfsJO+113366i3y>>) y>>S00 0 0 !OJ #s9~~ 5 #,/FF z+A? S SSr2c t|dD]}|jd|j}t||d}t |t |jkrt d|ddt d|d|D]M}dt|jpd }t d |j d |d |j Nd S)zBPretty-print selected cache revisions during confirmation prompts.cB|j|jfSN repo_typerepo_idlowerrTs r3z0print_cache_selected_revisions..zs$.RVR^RdRdRfRfAgr2key/c|jSrX commit_hashrevs r3r^z0print_cache_selected_revisions..|3?r2 - z (entire repo): z (detached)z z [z] N) sortedkeysrZr[rQr&printjoinr:rdsize_on_disk_str)rMrTrepo_keyr&revisionr:s r3print_cache_selected_revisionsrrxs)',,..4g4ghhh X Xn55t|55+D17R7RSSS y>>S00 0 0 1111 2 2 2   X   !!!! X XH88F8=1122BlD V8/VV4VV8;TVV W W W W X X Xr2 hf_cache_infoci}i}|jD]H}|j}|||<|jD] }||f||j<!I||fS)zPCreate lookup tables so CLI commands can resolve repo ids and revisions quickly.)reposcache_idr\r&rd)rs repo_lookuprevision_lookuprTrprqs r3build_cache_indexrys.0KLNO#MM=&&(( $ H M MH=A8.sQUQ]QcQcQeQe@fr2r_c&h|]}|jD]}|Sr1)r:).0rqrefs r3 z(collect_cache_entries..s*(e(e(eW_Wd(e(ePS(e(e(e(er2c|jSrXrcres r3r^z'collect_cache_entries..rgr2NcJ|dj|d |djndfS)Nrr)rvrdentrys r3r^z'collect_cache_entries..s+a!(-a(<a$$"r2c|djS)Nr)rvrs r3r^z'collect_cache_entries..s uQx'8r2)rkrur,r&appendsort)rsrzentries repo_refs_map sorted_reposrTrqs r3collect_cache_entriesrs!#G!#M--3f3fgggL))'(e(eT^(e(e(eff d  )"4>7R7RSSS 1 1h/0000 1 NND$< ( ( ( (:        88 999 M !!r2exprrc ` t|}|std|d|d|d|d t vr+tdd|dtt d tvr+td d |dttd d krt   fd SdvrRt  dtdttdtdtf fd }|Sdkr\ dkrtdddtdttdtdtffd }|Sdkrtdd dtdttdtdtf fd }|S)zqConvert a `hf cache ls` filter expression into the yes/no test we apply to each cache entry before displaying it.zInvalid filter expression: 'z'.r`opvaluezUnsupported operator 'z ' in filter ''. Must be one of .zUnsupported filter key '' in 'r;cBt||jn|jSrX)_compare_numeric size_on_disk)rTrq_rsize_thresholds r3r^z&compile_cache_filter..s+)9%-%9H ! !t?P  * * r2>r=r>rTrqnowrNcndkr|jn||jn|j}|dSt||z S)Nr=F) last_accessed last_modifiedr)rTrqr timestampr`rsecondss r3 _time_filterz*compile_cache_filter.._time_filters\*$$""'++'  u#C)ORAA Ar2r<r5z/Only '=' is supported for 'type' filters. Got 'rc>|jkSrX)rZr\)rTrqrexpecteds r3 _type_filterz*compile_cache_filter.._type_filters>''))X5 5r2z.Only '=' is supported for 'refs' filters. Got c||jn!|t}d|DvS)Nc6g|]}|Sr1)r\)r~rs r3 z>compile_cache_filter.._refs_filter..s (E(E(E(E(E(Er2)r:getr,r\)rTrqrr:r value_raws r3 _refs_filterz*compile_cache_filter.._refs_filtersO$,$88==m>O>OPTV_VaVa>b>bD??$$(E(E(E(E(EE Er2)_FILTER_PATTERNmatchstrip ValueErrorgroupr\_ALLOWED_OPERATORSlist _FILTER_KEYSrrrr rfloatbool) rrrrrrrr`rrrrs ` @@@@@@r3compile_cache_filterrs  ! !$**,, / /E B@@@@AAA ++e   " " $ $C T  B G$$**,,I ###v"vv4vv[_`r[s[svvvwww ,lClltllW[\hWiWilllmmm f}}#I..       &&& !2!233 B~ BBT9U B\a Bfj B B B B B B B B f}}??$$ 99UrUUUVV V 6~ 6BT9U 6Z_ 6dh 6 6 6 6 6 6 99SbSSSTT T F~ FBT9U FZ_ Fdh F F F F F F Fr2rc g}|D]\}}|rQ| |j|j|jt|j|j|j|jt|j d}nO|j|j|j|j|jt| |td}| ||S)zGNormalize cache entries into serializable records for JSON/CSV exports.N)r[rZrq snapshot_pathrrrr:)r[rZrrrr:) r[rZrdr-rrrrrkr:rr,r)rrzrpayloadrTrqrecords r3_build_cache_export_payloadrs%'G!h  <!^$0!$X%;!.sp    h# $)//22* ..//  $##r2)rr LAST_ACCESSEDrrcg|]u\}}|j|jd|jpd|jdt |tgvS)rrrj) rvrorlast_accessed_strrrnrkrr,)r~rTrrs r3rz-print_cache_entries_table..+s    a  %++A..&,"& 1 1$ D DEEFF      r2)headersch|]\}}|Sr1r1)r~rTrs r3rz,print_cache_entries_table..8s000WT1D000r2c3$K|] \}}|dV dS)Nrr1r~rrqs r3 z,print_cache_entries_table..;s+RR;1hX=QQ=Q=Q=Q=QRRr2c3.K|]\}}||jVdSrXrrs r3rz,print_cache_entries_table..<s.bb;1hXMa.MaMaMaMabbr2c3>K|]}t|jVdSrX)rQr&r~rTs r3rz,print_cache_entries_table..>s,JJTS00JJJJJJr2c3$K|] }|jV dSrXrrs r3rz,print_cache_entries_table..?s%DDt*DDDDDDr2z Found z repo(s) for a total of z revision(s) and z on disk.)rmrrQsumrrbold) rrzrmessager table_rows unique_reposrHrevision_count total_sizesummarys ` r3print_cache_entries_tablers 2Ch..Ih g EEE   #*    KJJ     #     (:w / / /00000000L\""JERR'RRRRRbb'bbbbb JJ\JJJJJDD|DDDDD BBB^BB^jku^v^vBBBG $)G  r2ct|||}tj|tjdtjddS)z7Dump cache entries as JSON for scripting or automation.rzrr)indent N)rjsondumpsysstdoutwrite)rrzrrs r3print_cache_entries_jsonrEsO*'EVfstttGIgsz!,,,,JTr2leftrrightc|dS||k||k||k||k||k||kd}||vrtd|||S)z)Evaluate numeric comparisons for filters.NF)r5r7r6r4r9r8z)Unsupported numeric comparison operator: )r)rrr comparisonss r3rrNsr |uU]em E\ E\emem K IRIIJJJ r?r2 sort_expr.c t|}|st d|d|d|d}t vr+t dd|dtt d|r|n t}|d k}d td ttd fffd }||fS)zConvert a `hf cache ls` sort expression into a key function for sorting entries. Returns: A tuple of (key_function, reverse_flag) where reverse_flag indicates whether to sort in descending order (True) or ascending order (False). zInvalid sort expression: 'z5'. Expected format: 'key' or 'key:asc' or 'key:desc'.r`orderzUnsupported sort key 'rrrr@rrN.c2|\}}dkr|j}|fSdkr||jn|j}|fSdkr|j|jnd}|fSdkr&||j|jnd}n|j|jnd}|fSt d)Nr?r;r=gr>zUnsupported sort key: )rvr\rrrr)rrTrqrr`s r3 _sort_keyz%compile_cache_sort.._sort_keywsh &==,,..E8O &==-5-AH))tGXE8O *  +/*<*HD&&cE8O *  #2:2H2T..Z].2.@.L**RU8O7#77888r2) _SORT_PATTERNrrr\rr _SORT_KEYSr_SORT_DEFAULT_ORDER CacheEntryr0r)rrexplicit_orderrreverserr`s @r3compile_cache_sortrbs     1 1 7 7 9 9 : :E xvivvvwww ++e   " " $ $C[[))N *m#mmYmmZ^_iZjZjmmmnnn- JNN2Ec2JEvoG99c3h999999< g r2targetsc8t|\}}tt}t}g}|D]}|}|s|} t jd| rh|| } | ||p| \} } ||  | | | j || } | ||| j D]7} ||  | | | j 8d| D}tt||t|S)z8Resolve the deletion targets into a deletion resolution.z[0-9a-fA-F]{40}Nc4i|]\}}|t|Sr1)r,)r~rTrevss r3 z-_resolve_deletion_targets..s$PPPttYt__PPPr2r&r'r()ryrsetrr\re fullmatchrraddrdr&rPr%r,r0)rsrrwrxr'r&r( raw_targettargetloweredrrTrq matched_repofrozen_selecteds r3_resolve_deletion_targetsrs#4]#C#C K>I#>N>NH%%IG00 !!##  ,,.. <*G 4 4 #''00E}z***"ND( TN  x ( ( ( MM(. / / / "w//   NN: & & & $. 0 0H \ " & &x 0 0 0 MM(. / / / / 0QPx~~?O?OPPPO I&& g   r2)z hf cache lszhf cache ls --revisionsz*hf cache ls --filter "size>1GB" --limit 20zhf cache ls --format json)examplesF cache_dirz9Cache directory to scan (defaults to Hugging Face cache).r&zCInclude revisions in the output instead of aggregated repositories.filterz-fz--filterzZFilter entries (e.g. 'size>1GB', 'type=model', 'accessed>7d'). Can be used multiple times.formatzOutput format.quietz-qz--quietz-Print only IDs (repo IDs or revision hashes).raSort entries by key. Supported keys: 'accessed', 'modified', 'name', 'size'. Append ':asc' or ':desc' to explicitly set the order (e.g., 'modified:asc'). Defaults: 'accessed', 'modified', 'size' default to 'desc' (newest/biggest first); 'name' defaults to 'asc' (alphabetical).limitzSLimit the number of results returned. Returns only the top N entries after sorting.cf t|}n*#t$r}td|j|d}~wwxYw|pg} t ||\}  fd| D} n4#t $r'}t jt||d}~wwxYwtj | D]fd| D} |rd t|j \} } | | | n4#t $r'}t jt||d}~wwxYw|(|dkrt jd|d | d|} |r'| D]"\}}t||jn|j#dSt jt$t jt(i}||| | S) z&List cached repositories or revisions.Cache directory not found: N)rzc0g|]}t|Sr1)r)r~rrs r3rzls.. s$TTTD*4??TTTr2cFg|]}|d|d|S)rrr1)r~rfnrs r3rzls..s4MMMUE!HeAh1L1LM5MMMr2)r`rrz&Limit must be a positive integer, got rr)rrrrrrtyper BadParameterr-timerrrrmrdrvrtablerrr)rr&rrrrrrsexcfiltersr filter_fns sort_key_fnrrTrq formattersr rrs @@@r3lsrs;xO&y11 OOODS]DDEE3NOlG2=T]^^^G]4TTTTGTTT 444 S**34 )++CNNMMMMMgMMM 8 8#5dj#A#A K LL['L : : : : 8 8 8$SXX..C 7 8  199$%Ve%V%V%VWW W&5&/ % S SND( (*>(&&DM R R R R 53J :f gR_ ` ` ``s@ <7<A'' B1"BB.C44 D%>"D  D%)zhf cache rm model/gpt2zhf cache rm z hf cache rm model/gpt2 --dry-runzhf cache rm model/gpt2 --yeszQOne or more repo IDs (e.g. model/bert-base-uncased) or revision hashes to delete.yesz-yz--yeszSkip confirmation prompt.dry_runz,Preview deletions without removing anything.c J t|}n*#t$r}td|j|d}~wwxYwt ||}|jr+t d|jD]}t d|t|jdkr$t dtj d|j t|j}t|j} g} | jr| | jd| jr| | jd | s| | jd d | } t d | d |jd t+|j|rt ddS|s'tjddst ddS|t|j} t d| jd| jd|jd dS)z(Remove cached repositories or revisions.r Nz2Could not find the following targets in the cache:rhrzNothing to delete.codez repo(s)z revision(s)z and About to delete z totalling rDry run: no files were deleted.zProceed with deletion?F)defaultzDeletion cancelled.Deleted z repo(s) and z revision(s); freed )rrrrrr(rmrQr&rExitdelete_revisionsrkrUr'rHrrIrJrnexpected_freed_size_strrrconfirmexecute) rrrrrsr resolutionrstrategycounts summary_parts summary_texts r3rmr++sJO&y11 OOODS]DDEE3NO+='BBJ" BCCC' " "E ... ! ! ! ! :   A%% "###ja    -}-vj6J/K/KLH !4 5 5F!M = 1;;;<<< $M =KKKLLL K ;IIIJJJ<< ..L Y\ Y Yh6V Y Y YZZZ":#6777 /000 u}%=uMMM #$$$  !4 5 5F  H6$ H H63N H HdleE H H H 949zhf cache prunezhf cache prune --dry-runcP t|}n*#t$r}td|j|d}~wwxYwi}t }|jD]G}t d|jD}|s#|||<|d|DHt|dkrtddStt ||d} |j t| j} t|} td | jd | jd t#||rtd dS|s%t%jd stddS| td| jd| jddS)z)Remove detached revisions from the cache.r Nc3JK|]}t|jdk|VdS)rN)rQr:r~rqs r3rzprune..s7``(HMHZHZ^_H_H_XH_H_H_H_``r2c3$K|] }|jV dSrXrcr/s r3rzprune..s%GG(-GGGGGGr2rz2No unreferenced revisions found. Nothing to prune.r1rrz unreferenced revision(s) (z total).rzProceed?zPruning cancelled.r z! unreferenced revision(s); freed r)rrrrrrur,r&updaterQrmr%r"rkrUrJr#rrrr$r%) rrrrsrr'r&rTdetachedr&r'r(s r3pruner3~s0O&y11 OOODS]DDEE3NOEGH%%I#HH``dn`````  !GGhGGGGGGG 9~~ BCCC$I&&J .}-vj6J/K/KLH  * *F }66}}S[Ss}}}#8,,, /000 u}Z00 "###  vV0 v vS[Ss v v vwwwwwr,)zhf cache verify gpt2z)hf cache verify gpt2 --revision refs/pr/1z.hf cache verify my-dataset --repo-type datasetr[rZrqzXCache directory to use when verifying files from cache (defaults to Hugging Face cache). local_dirz?If set, verify files under this directory instead of the cache.fail_on_missing_filesz--fail-on-missing-filesz?Fail if some files exist on the remote but are missing locally.fail_on_extra_filesz--fail-on-extra-fileszLFail if some files exist locally but are not present on the remote revision.tokenc V|&|$tdtjdt|}||t |dr|jnt|||||} d} t| j } | rNtd | j D]5} td | d d | d d| dd| d6d} | j rY|r.td| j D]} td | d} n)t| j d}td|| j rY|r.td| j D]} td | d} n)t| j d}td|| j }| dkrJtd|d|jd|dtd| jtj| td| jd|d|jd|tddS) aVerify checksums for a single repo revision from cache or a local directory. Examples: - Verify main revision in cache: `hf cache verify gpt2` - Verify specific revision: `hf cache verify gpt2 --revision refs/pr/1` - Verify dataset: `hf cache verify karpathy/fineweb-edu-100b-shuffle --repo-type dataset` - Verify local dir: `hf cache verify deepseek-ai/DeepSeek-OCR --local-dir /path/to/repo` NzCCannot pass both --local-dir and --cache-dir. Use one or the other.rr)r7r)r[rZrqr4rr7ru;❌ Checksum verification failed for the following file(s):rhpathz : expected rz ( algorithmz), got actualrz1Missing files (present remotely, absent locally):zM remote file(s) are missing locally. Use --fail-on-missing-files for details.u⚠️ z/Extra files (present locally, absent remotely):zV local file(s) do not exist on the remote repo. Use --fail-on-extra-files for details.u❌ Verification failed for 'z' (z) in rz Revision: u ✅ Verified z file(s) for 'z All checksums match.)rmrr!r verify_repo_checksumshasattrrr-r mismatches missing_pathsrQ extra_paths verified_pathrq checked_count)r[rZrqrr4r5r6r7apiresult exit_codehas_mismatchesmpwarningverified_locations r3verifyrKs`!6 STTTja    5 ! ! !C  & &%,Y%@%@T)//c)nn 'FI&+,,N KLLL" f fA d6ddq}dd+ddWXYaWbdd e e e e   ( ( E F F F) " "jQjj!!!!IIv+,,;;;  &W&& ' ' '  (  ( C D D D' " "jQjj!!!!IIv)**999  &W&& ' ' ',A~~ dgdd)/ddPadddeee .V_..///ji(((( s&. s sg s s)/ s s`q s sttt "#####r2)NFF)WrKrrrr collectionsr dataclassesrenumrtypingrrrr r r r r rhuggingface_hub.errorsrutilsrrrrrrrrutils._parsingrr _cli_utilsrrrrrr r! cache_clir%compilerrrrrr_sort_options_dictrkr`r-r)rDrGrr, RepoRefsMaprUrrryrrrrrrrrr0rrrcommandrOptionrLrArgumentr+r3modelrKr1r2r3r\s ON ######!!!!!!QQQQQQQQQQQQQQQQQQQQ ++++++                    87777777 M> ? ? ?  $ "*abb666??? 5 5 5  LMM     6*  66C!s*-|||#|||$+. #}}}%%d="43xPPP  $>8,>#?? @ >9S>12 Tni8J.KKLTTTTT$ XW^YWiMj=j5k Xpt X X X X(( n eN$667 78:(((("""6:" 4 [ ()""""4> >)> ~x(:;UCTIJ>>>>B * 59JU $sCx.@/ * /59/JU/ ////d * 59JU 8E?Ed(3#3%*uSRUX9V0WY]0]*^3333l&[&49&Qd&&&&X     c\a\a  L    \a  V    \a c  m     \a*  !     +\a6    @     7\aF  7     G\aX   f     Y\ad e\a\a\a\a~ (   5HH  S d    H  L    H    ,     H*  ?    +H6 7HHHHV -/IJKK   )?x?x  L    ?x    ,     ?x  ?    ?x* +?x?x?xLK?xD ).   =c$c$ c$c$c$  k     c$  R    c$ %  %R    !c$.#  #_    /c$< =c$> ?c$c$c$c$c$c$r2