
    ZiP              '          d Z ddlZddlZddlZddlZddlZddlZddlmZm	Z	 ddl
m
Z
mZ ddlmZ ddlmZmZmZmZmZmZ ddlmZmZ dd	lmZ dd
lmZmZmZmZ ddlm Z  erddl!m"Z"  ej#        e$          Z%dZ&dZ'de(de)e(e(f         fdZ*e G d d                      Z+e G d d                      Z,e G d d                      Z- ed           G d d                      Z.e G d d                      Z/e G d d                      Z0e G d  d!                      Z1de(de)e(e(f         fd"Z2de(de3fd#Z4e G d$ d%                      Z5e G d& d'                      Z6 G d( d)          Z7d*e(de8e)e(e(f                  fd+Z9d,e(dee)e(e:e;f                  fd-Z<d.d/d0e(d1e(dee)e(e:e;ef                  fd2Z=d3e;de(fd4Z>dd5de(d6ed7         d8e:d9e;d:e:d;e;d<e(d=e(d>e3d?e3d@e3dAee         de5fdBZ?	 	 	 	 	 	 	 d\dDe(dEe(d.d/dFe3d?e3d>e3dGe3d@e3dHee7         dIee         de6fdJZ@dKe6ddfdLZAdKe6dMe(ddfdNZBdMe(de6fdOZCd]dKe6d.d/dPe3dIee         ddf
dQZDdKe6ddfdRZE	 	 d^dCdCdCdCdCddddddCdCdCddSdDee(         dEee(         d.d/dFe3d?e3d>e3dGe3d@e3dTee8e(                  dUee8e(                  dVee(         dKee(         dWee(         dXe3dPe3dYe3dZee3e(df         de6f$d[ZFdS )_zzShared logic for bucket operations.

This module contains the core buckets logic used by both the CLI and the Python API.
    N)	dataclassfield)datetimetimezone)Path)TYPE_CHECKINGAnyIteratorLiteralOptionalUnion   )	constantslogging)BucketNotFoundError)XetFileDatadisable_progress_barsenable_progress_barsparse_datetime)
StatusLineHfApihf://buckets/  pathreturnc                    |                      dd          }t          |          dk     s|d         r|d         st          d|  d          |d          d|d          }t          |          dk    r|d         nd}||fS )zSplit 'namespace/name(/optional/prefix)' into ('namespace/name', 'prefix').

    Returns (bucket_id, prefix) where prefix may be empty string.
    Raises ValueError if path doesn't contain at least namespace/name.
    /   r   r   zInvalid bucket path: 'z)'. Expected format: namespace/bucket_name )splitlen
ValueError)r   parts	bucket_idprefixs       S/root/projects/butler/venv/lib/python3.11/site-packages/huggingface_hub/_buckets.py_split_bucket_id_and_prefixr(   5   s     JJsAE
5zzA~~U1X~U1X~a$aaabbb8((eAh((IUaU1XXRFf    c                   L    e Zd ZU dZeed<   eed<   eed<   eed<   eed<   d Z	dS )	
BucketInfoa  
    Contains information about a bucket on the Hub. This object is returned by [`bucket_info`] and [`list_buckets`].

    Attributes:
        id (`str`):
            ID of the bucket.
        private (`bool`):
            Is the bucket private.
        created_at (`datetime`):
            Date of creation of the bucket on the Hub.
        size (`int`):
            Size of the bucket in bytes.
        total_files (`int`):
            Total number of files in the bucket.
    idprivate
created_atsizetotal_filesc                 H   |                     d          | _        |                     d          | _        t          |                     d                    | _        |                     d          | _        |                     d          | _         | j        j        di | d S )Nr,   r-   	createdAtr/   
totalFiles )	popr,   r-   r   r.   r/   r0   __dict__updateselfkwargss     r'   __init__zBucketInfo.__init__[   s    **T""zz),,(K)@)@AAJJv&&	!::l33&&v&&&&&r)   N)
__name__
__module____qualname____doc__str__annotations__boolr   intr;   r4   r)   r'   r+   r+   C   sd            	GGGMMM
III' ' ' ' 'r)   r+   c                       e Zd ZU eeeef         ed<   eed<    ed          Z	e
e         ed<    ed          Ze
e         ed<    ed          Zeed	<    ed          Ze
e         ed
<   ddZdS )_BucketAddFilesourcedestinationN)defaultxet_hashr/   Finitmtimecontent_typer   c                    d | _         t          | j        t          t          f          r$t          j        | j                  d         | _         | j         $t          j        | j                  d         | _         t          t          | j        t                    s't          j                            | j                  dz  nt          j                    dz            | _        d S )Nr   r   )rM   
isinstancerF   r@   r   	mimetypes
guess_typerG   rC   bytesosr   getmtimetimerL   )r9   s    r'   __post_init__z_BucketAddFile.__post_init__n   s     dkC;// 	E ) 4T[ A A! DD$ ) 4T5E F Fq ID8B4;PU8V8VnBGT[))D00\`\e\g\gjn\n
 



r)   r   N)r<   r=   r>   r   r@   r   rR   rA   r   rI   r   r/   rC   rL   rM   rV   r4   r)   r'   rE   rE   d   s         #tU"#####eD111Hhsm111%---D(3----E"""E3""""'%U"3"3"3L(3-333	
 	
 	
 	
 	
 	
r)   rE   c                       e Zd ZU eed<   dS )_BucketDeleteFiler   N)r<   r=   r>   r@   rA   r4   r)   r'   rY   rY   z   s         
IIIIIr)   rY   T)frozenc                   (    e Zd ZU dZeed<   eed<   dS )BucketFileMetadataa%  Data structure containing information about a file in a bucket.

    Returned by [`get_bucket_file_metadata`].

    Args:
        size (`int`):
            Size of the file in bytes.
        xet_file_data (`XetFileData`):
            Xet information for the file (hash and refresh route).
    r/   xet_file_dataN)r<   r=   r>   r?   rC   rA   r   r4   r)   r'   r\   r\      s3         	 	 IIIr)   r\   c                       e Zd ZU dZeed<   dZeed<    ed          Zeed<    ed          Z	eed<    ed          Z
eed	<   ddZdS )	BucketUrla  Describes a bucket URL on the Hub.

    `BucketUrl` is returned by [`create_bucket`]. At initialization, the URL is parsed to populate properties:
    - endpoint (`str`)
    - namespace (`str`)
    - bucket_id (`str`)
    - url (`str`)
    - handle (`str`)

    Args:
        url (`str`):
            String value of the bucket url.
        endpoint (`str`, *optional*):
            Endpoint of the Hub. Defaults to <https://huggingface.co>.
    urlr    endpointFrJ   	namespacer%   handler   Nc                    | j         pt          j        | _         | j                            | j         d                              d          }|                    d          r|t          d          d          }t          |          \  }}|rt          d| j                   |
                    d          d         | _        || _        d| j         | _        d S )Nr    r   zbuckets/zUnable to parse bucket URL: r   r   )ra   r   ENDPOINTr`   replacestrip
startswithr"   r(   r#   r!   rb   r%   rc   )r9   url_pathr%   r&   s       r'   rV   zBucketUrl.__post_init__   s    ;); 8##DM266<<SAAz** 	3J 1 12H7AA	6 	HFDHFFGGG"--a0"6dn66r)   rW   )r<   r=   r>   r?   r@   rA   ra   r   rb   r%   rc   rV   r4   r)   r'   r_   r_      s            
HHHHcU&&&Is&&&U&&&Is&&&%U###FC###7 7 7 7 7 7r)   r_   c                   d    e Zd ZU dZed         ed<   eed<   eed<   eed<   ee	         ed<   d Z
d	S )

BucketFilez
    Contains information about a file in a bucket on the Hub. This object is returned by [`list_bucket_tree`].

    Similar to [`RepoFile`] but for files in buckets.
    filetyper   r/   rI   rL   c                 2   |                     d          | _        |                     d          | _        |                     d          | _        |                     d          | _        |                     dd           }|rt          |          nd | _        d S )Nrm   r   r/   xetHashrL   )r5   rm   r   r/   rI   r   rL   )r9   r:   rL   s      r'   r;   zBucketFile.__init__   s~    JJv&&	JJv&&	JJv&&	

9--

7D)).3=^E***


r)   N)r<   r=   r>   r?   r   rA   r@   rC   r   r   r;   r4   r)   r'   rk   rk      sm           &/
III
IIIMMMH> > > > >r)   rk   c                   D    e Zd ZU dZed         ed<   eed<   eed<   d ZdS )BucketFolderz
    Contains information about a directory in a bucket on the Hub. This object is returned by [`list_bucket_tree`].

    Similar to [`RepoFolder`] but for directories in buckets.
    	directoryrm   r   uploaded_atc                     |                     d          | _        |                     d          | _        t          |                     d                    | _        d S )Nrm   r   
uploadedAt)r5   rm   r   r   rs   r8   s     r'   r;   zBucketFolder.__init__   sJ    JJv&&	JJv&&	)&**\*B*BCCr)   N)	r<   r=   r>   r?   r   rA   r@   r   r;   r4   r)   r'   rq   rq      s\           +

IIID D D D Dr)   rq   c                     |                      t                    st          d|  dt                     t          |                     t                              S )zParse a bucket path like hf://buckets/namespace/bucket_name/prefix into (bucket_id, prefix).

    Returns:
        tuple: (bucket_id, prefix) where bucket_id is "namespace/bucket_name" and prefix may be empty string.
    zInvalid bucket path: z. Must start with )rh   BUCKET_PREFIXr#   r(   removeprefixr   s    r'   _parse_bucket_pathrz      sT     ??=)) ZXXXXXYYY&t'8'8'G'GHHHr)   c                 6    |                      t                    S )z!Check if a path is a bucket path.)rh   rw   ry   s    r'   _is_bucket_pathr|      s    ??=)))r)   c                       e Zd ZU dZed         ed<   eed<   dZee	         ed<   dZ
eed<   dZee         ed	<   dZee         ed
<   dZee         ed<   dS )SyncOperationz,Represents a sync operation to be performed.)uploaddownloaddeleteskipactionr   Nr/   r    reasonlocal_mtimeremote_mtimebucket_file)r<   r=   r>   r?   r   rA   r@   r/   r   rC   r   r   r   r   rk   r4   r)   r'   r~   r~      s         66:;;;;
IIID(3-FC!%K#%%%"&L(3-&&&(,K*%,,,,,r)   r~   c                       e Zd ZU dZeed<   eed<   eed<    ee          Zee	         ed<   de
eeeef         f         fdZd	S )
SyncPlanz Represents a complete sync plan.rF   dest	timestamp)default_factory
operationsr   c                 >   t          d | j        D                       }t          d | j        D                       }t          d | j        D                       }t          d | j        D                       }t          d | j        D                       }|||||dS )Nc              3   0   K   | ]}|j         d k    dV  dS )r   r   Nr   .0ops     r'   	<genexpr>z#SyncPlan.summary.<locals>.<genexpr>  .      KKBRY(5J5Ja5J5J5J5JKKr)   c              3   0   K   | ]}|j         d k    dV  dS )r   r   Nr   r   s     r'   r   z#SyncPlan.summary.<locals>.<genexpr>  s.      OObryJ7N7N7N7N7N7NOOr)   c              3   0   K   | ]}|j         d k    dV  dS )r   r   Nr   r   s     r'   r   z#SyncPlan.summary.<locals>.<genexpr>  r   r)   c              3   0   K   | ]}|j         d k    dV  dS )r   r   Nr   r   s     r'   r   z#SyncPlan.summary.<locals>.<genexpr>  s.      GG"293F3FA3F3F3F3FGGr)   c              3   :   K   | ]}|j         d v |j        pdV  dS )r   r   r   N)r   r/   r   s     r'   r   z#SyncPlan.summary.<locals>.<genexpr>  s5      gg"29PfCfCfACfCfCfCfggr)   )uploads	downloadsdeletesskips
total_size)sumr   )r9   r   r   r   r   r   s         r'   summaryzSyncPlan.summary  s    KK$/KKKKKOODOOOOOO	KK$/KKKKKGGGGGGGggggggg
"$
 
 	
r)   N)r<   r=   r>   r?   r@   rA   r   listr   r~   dictr   rC   r   r4   r)   r'   r   r   	  s         **KKK
IIINNN&+eD&A&A&AJ]#AAA
c5c?23 
 
 
 
 
 
r)   r   c                       e Zd ZdZ	 	 	 d
deee                  deee                  deeeeef                           fdZdede	fd	Z
dS )FilterMatcherz4Matches file paths against include/exclude patterns.Ninclude_patternsexclude_patternsfilter_rulesc                 <    |pg | _         |pg | _        |pg | _        dS )a  Initialize the filter matcher.

        Args:
            include_patterns: Patterns to include (from --include)
            exclude_patterns: Patterns to exclude (from --exclude)
            filter_rules: Rules from filter file as list of ("+"/"-", pattern) tuples
        Nr   r   r   )r9   r   r   r   s       r'   r;   zFilterMatcher.__init__)  s1     !1 6B 0 6B(.Br)   r   r   c                     | j         D ]"\  }}t          j        ||          r|dk    c S #| j        D ]}t          j        ||          r dS | j        D ]}t          j        ||          r dS | j        rdS dS )zCheck if a path should be included based on the filter rules.

        Filtering rules:
        - Filters are evaluated in order, first matching rule decides
        - If no rules match, include by default (unless include patterns are specified)
        +FT)r   fnmatchr   r   )r9   r   signpatterns       r'   matcheszFilterMatcher.matches:  s     ". 	# 	#MD'tW-- #s{"""# , 	 	GtW-- uu , 	 	GtW-- tt   	5 tr)   )NNN)r<   r=   r>   r?   r   r   r@   tupler;   rB   r   r4   r)   r'   r   r   &  s        >> 15048<	/ /"49-/ #49-/ tE#s(O45	/ / / /"C D      r)   r   filter_filec                    g }t          |           5 }|D ]}|                                }|r|                    d          r.|                    d          r2|                    d|dd                                         f           u|                    d          r2|                    d|dd                                         f           |                    d|f           	 ddd           n# 1 swxY w Y   |S )zParse a filter file and return a list of (sign, pattern) tuples.

    Filter file format:
    - Lines starting with "+" are include patterns
    - Lines starting with "-" are exclude patterns
    - Empty lines and lines starting with "#" are ignored
    #r   r   N-)openrg   rh   append)r   rulesflines       r'   _parse_filter_filer   W  sJ    E	k		 *a 
	* 
	*D::<<D 4??3// s## *c48>>#3#345555%% *c48>>#3#345555 c4[))))
	** * * * * * * * * * * * * * * Ls   CC66C:=C:
local_pathc              #   :  K   t           j                            |           } t           j                            |           st	          d|            t          j        |           D ]\  }}}|D ]}t           j                            ||          }t           j                            ||           }|                    t           j	        d          }t           j        
                    |          }t           j                            |          dz  }|||fV  dS )zsList all files in a local directory.

    Yields:
        tuple: (relative_path, size, mtime_ms) for each file
    z Local path must be a directory: r   r   N)rS   r   abspathisdirr#   walkjoinrelpathrf   sepgetsizerT   )	r   root_filesfilename	full_pathrel_pathr/   mtime_mss	            r'   _list_local_filesr   t  s
      ,,J7==$$ JHJHHIII'*-- + +a 	+ 	+HT844Iwy*==H''44H7??9--Dw''	22T9HD(*****	++ +r)   apir   r%   r&   c              #     K   |                      ||pdd          D ]}t          |t                    r|j        }|r]|                    |dz             r|t          |          dz   d         }n,||k    r#d|v r|                    dd          d         n|}n~|}|j        r|j                                        dz  nd}||j	        ||fV  dS )	zList all files in a bucket with a given prefix.

    Yields:
        tuple: (relative_path, size, mtime_ms, bucket_file) for each file.
            bucket_file is the BucketFile object from list_bucket_tree.
    NT)r&   	recursiver   r   r   r   )
list_bucket_treerO   rq   r   rh   r"   rsplitrL   r   r/   )r   r%   r&   itemr   r   r   s          r'   _list_remote_filesr     s      $$Yv~QU$VV 2 2dL)) 	y  	v|,, Fa 1 1269Tkk4;;sA..r22t H48JE4:''))D00A	8T11111'2 2r)   r   c                 l    t          j        | dz  t          j                                                  S )z3Convert mtime in milliseconds to ISO format string.r   )tz)r   fromtimestampr   utc	isoformat)r   s    r'   _mtime_to_isor     s*    !(T/hlCCCMMOOOr)   )r   r   r   source_sizesource_mtime	dest_size
dest_mtimesource_newer_labeldest_newer_labelignore_sizesignore_timesignore_existingr   c                    t          |dk    r|n|          }t          |dk    r|n|          }| |||d}|
rt          dddd|S ||k    }||z
  t          k    }|r5|rt          d|||d|S ||z
  t          k    }|r|nd}t          dd|d|S |	r!|rt          d|d|d|S t          ddd	d|S |s|r|rdn|}t          d|||d|S t          ddd
d|S )a  Compare source and dest files and return the appropriate sync operation.

    This is a unified helper for both upload and download directions.

    Args:
        path: Relative file path
        action: "upload" or "download"
        source_size: Size of the source file (bytes)
        source_mtime: Mtime of the source file (milliseconds)
        dest_size: Size of the destination file (bytes)
        dest_mtime: Mtime of the destination file (milliseconds)
        source_newer_label: Label when source is newer (e.g., "local newer" or "remote newer")
        dest_newer_label: Label when dest is newer (e.g., "remote newer" or "local newer")
        ignore_sizes: Only compare mtime
        ignore_times: Only compare size
        ignore_existing: Skip files that exist on receiver
        bucket_file: BucketFile object (for downloads only)

    Returns:
        SyncOperation describing the action to take
    r   )r   r/   r   r   r   z&exists on receiver (--ignore-existing))r   r   )r   r   r   z
same mtimezsize differsz	same size	identicalNr4   )r   r~   _SYNC_TIME_WINDOW_MS)r   r   r   r   r   r   r   r   r   r   r   r   local_mtime_isoremote_mtime_isobase_kwargssize_differssource_newer
dest_newerskip_reasonr   s                       r'   _compare_files_for_syncr     s   H $Fh4F4FLLJWWO$6X3E3EZZ<XX &(	# #K  lkF3[kk_jkkk)+L :-1EEL S 	S r7IWbrrfqrrr$|37KKJ.8J**lK R{RRkRRR	 
S 	S n~S^nnbmnnn R{RRkRRR 	S< 	S'3K^^9KF fv;ffZefff R{RRkRRRr)   FrF   r   r   existingfilter_matcherstatusc
                    |pt                      }t          |            ot          |          }
t          |           ot          |           }|
s|st          d          t          | |t	          j        t          j                                                            }d}|
rt          j
                            |           }t          |          \  }}t          j
                            |          st          d|           i }t          |          D ]J\  }}}|                    |          r||f||<   |	r&|	                    dt#          |           d           K|	r&|	                    dt#          |           d           i }|	r,	 |                    |          j        }n# t*          $ r Y nw xY w	 t-          |||          D ]V\  }}}}|                    |          r||f||<   |	r1|d| nd}|	                    d	t#          |           | d           Wn.# t.          $ r! t0                              d
| d           Y nw xY w|	r&|	                    d	t#          |           d           t5          |                                          t5          |                                          z  }|	r&|	                    dt#          |           d           t9          |          D ]L}|                    |          }|                    |          }|r|s|rF|j                            tA          d||d         dtC          |d                                        y|j                            tA          d||d         dtC          |d                                        |r@|r>|\  }}|\  }}|j                            tE          |d||||dd|||                     |sI|rG|rE|j                            tA          d||d         dtC          |d                                        Nnt          |           \  }}t          j
                            |          }i }i } |	r,	 |                    |          j        }n# t*          $ r Y nw xY wt-          |||          D ][\  }}}}!|                    |          r||f||<   |!| |<   |	r1|d| nd}|	                    d	t#          |           | d           \|	r&|	                    d	t#          |           d           i }t          j
                            |          rZt          |          D ]J\  }}}|                    |          r||f||<   |	r&|	                    dt#          |           d           K|	r&|	                    dt#          |           d           t5          |                                          t5          |                                          z  }|	r&|	                    dt#          |           d           t9          |          D ]t}|                    |          }|                    |          }|r|s|rF|j                            tA          d||d         dtC          |d                                        y|j                            tA          d||d         dtC          |d                   |                     |                               |rT|rR|\  }}|\  }}|j                            tE          |d||||dd||||                     |                               )|sI|rG|rE|j                            tA          d||d         dtC          |d                                        v|S )zCompute the sync plan by comparing source and destination.

    Returns:
        SyncPlan with all operations to be performed
    z[One of source or dest must be a bucket path (hf://buckets/...) and the other must be local.rF   r   r   NzSource must be a directory: zScanning local directory (z files)r   r    zScanning remote bucket (zBucket 'z' not found, treating as empty.zComparing files (z paths)r   r   znew file (--existing)r   )r   r   r/   r   r   r   znew filezlocal newerzremote newer)r   r   r   r   r   r   r   r   r   r   r   r   znot in source (--delete))r   r   r/   r   r   r   )r   r   r/   r   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   )#r   r|   r#   r   r   nowr   r   r   rS   r   r   rz   r   r   r   r7   r"   donebucket_infor0   	Exceptionr   r   loggerdebugsetkeyssortedgetr   r   r~   r   r   )"rF   r   r   r   r   r   r   r   r   r   	is_uploadis_downloadplanremote_totalr   r%   r&   local_filesr   r/   r   remote_filesr   	total_str	all_pathsr   
local_inforemote_info
local_sizer   remote_sizer   bucket_file_mapr   s"                                     r'   _compute_sync_planr    s	   " $6}N#F+++E0E0EI!&))G/$2G2G.GK x[ xvwww,x|,,6688  D #'L BW__V,,
.t44	6w}}Z(( 	JHJHHIII (9*(E(E 	V 	V$HdH%%h// 9)-x(8H% VT3{;K;KTTTUUU 	PKKNS5E5ENNNOOO 	"y99E   		P/A#yRX/Y/Y d d+$!!))(33 >.2H-=L* d6B6N 2L 2 2 2TVIMM"bS=N=N"bPY"b"b"bcccd # 	P 	P 	PLLNINNNOOOOO	P  	OKKM3|3D3DMMMNNN ((**++c,2C2C2E2E.F.FF	 	EKKCC	NNCCCDDD9%% 8	 8	D$..J&**400K 4+ 4 O**%#)!%!+A#:(5jm(D(D      O**%#+!%!+A#-(5jm(D(D        *4'
K,7)\&&+!'$.%0"-#/+8)7%1%1(7        
K 
F 
&&!'!(^9%2;q>%B%B    a8	x /v66	6W__T**
 *, 	"y99E   5GYX^5_5_ 	` 	`1HdHk%%h// 8*.)9X&,7) `2>2J....PR	^\9J9J^I^^^___ 	OKKM3|3D3DMMMNNN7==$$ 	Z,=j,I,I Z Z($!))(33 =-18,<K) ZMM"Xs;?O?O"X"X"XYYY 	PKKNS5E5ENNNOOO ))++,,s;3C3C3E3E/F/FF	 	EKKCC	NNCCCDDD9%% :	 :	D&**400K$..J 6: 6 O**%#)!%!,Q#:)6{1~)F)F      O**%#-!%!,Q#-)6{1~)F)F(7(;(;D(A(A  	 	 	 	   ,7)\*4'
K&&+!)$/%1",#.+9)6%1%1(7$3$7$7$=$=       ! 
Z 
F 
&&!'!']9$1*Q-$@$@     Ks7   F- -
F:9F:>A(H' '(II?R 
R'&R'r   c                    d| j         | j        | j        |                                 d}|                    t          j        |          dz              | j        D ]u}d|j        |j	        |j
        d}|j        
|j        |d<   |j        
|j        |d<   |j        
|j        |d	<   |                    t          j        |          dz              vdS )
z1Write a sync plan as JSONL to a file-like object.header)rm   rF   r   r   r   
	operation)rm   r   r   r   Nr/   r   r   )rF   r   r   r   writejsondumpsr   r   r   r   r/   r   r   )r   r   r  r   op_dicts        r'   _write_planr    s     +	^<<>> F GGDJv%&&& o , ,iGi	#
 #
 7 gGFO>%%'^GM"?&&(oGN#	
7##d*++++, ,r)   	plan_filec                 x    t          |d          5 }t          | |           ddd           dS # 1 swxY w Y   dS )z!Save a sync plan to a JSONL file.wN)r   r  )r   r  r   s      r'   
_save_planr     s    	i		 D!                 s   /33c                    t          |           5 }|                                }ddd           n# 1 swxY w Y   |st          d|            t          j        |d                   }|                    d          dk    rt          d          t          |d         |d         |d	         
          }|dd         D ]}t          j        |          }|                    d          dk    r0|j                            t          |d         |d         |                    d          |                    dd          |                    d          |                    d                               |S )z#Load a sync plan from a JSONL file.NzEmpty plan file: r   rm   r  z0Invalid plan file: expected header as first linerF   r   r   r   r   r  r   r   r/   r   r    r   r   )r   r   r/   r   r   r   )
r   	readlinesr#   r  loadsr   r   r   r   r~   )r  r   linesr  r   r   r  s          r'   
_load_planr    s   	i A                :8Y88999 Za!!Fzz&X%%KLLLhF^%  D abb	 
 
*T"";;v+--x(V_[[(({{8R00#KK66$[[88  		
 		
 		
 		
 Ks   155verbosec                 
   t          | j                   ot          | j                  }t          | j                  ot          | j                   }|rt          j                            | j                  }t          | j                  \  }}|                    d          }g }	g }
| j        D ]}|j	        dk    rtt          j        
                    ||j                  }|r| d|j         n|j        }|r t          d|j         d|j         d           |	                    ||f           |j	        dk    rM|r| d|j         n|j        }|r t          d|j         d|j         d           |
                    |           |j	        dk    r"|r t          d	|j         d|j         d           	|	s|
r|rg }|	r&|                    d
t          |	           d           |
r&|                    dt          |
           d           |                    d
                    |                                                     |                    ||	pd|
pd           dS dS |rt          | j                  \  }}|                    d          }t          j                            | j                  }t          j        |d           g }g }| j        D ]o}|j	        dk    rt          j        
                    ||j                  }t          j        t          j                            |          d           |r t          d|j         d|j         d           |j        |                    |j        |f           |r| d|j         n|j        }|                    ||f           |j	        dk    r^t          j        
                    ||j                  }|r t          d|j         d|j         d           |                    |           B|j	        dk    r"|r t          d	|j         d|j         d           qt          |          dk    r>|r&|                    dt          |           d           |                    ||           |r(|r&|                    dt          |           d           |D ]}t          j                            |          rt          j        |           t          j                            |          }||k    rK	 t          j        |           t          j                            |          }n# t2          $ r Y n
w xY w||k    KdS dS )zExecute a sync plan.r   r   z  Uploading: z ()r   z  Deleting: r   z  Skipping: z
uploading z filesz	deleting z, N)addr   T)exist_okr   z  Downloading: r   zDownloading z	Deleting z local files)r|   rF   r   rS   r   r   rz   rstripr   r   r   printr   r   r"   r   
capitalizebatch_bucket_filesmakedirsdirnamer   download_bucket_filesexistsremovermdirOSError)r   r   r  r   r   r   r   r%   r&   	add_filesdelete_pathsr   
local_fileremote_pathr$   download_filesdelete_files	file_pathparents                      r'   _execute_planr6  1  s   #DK000O_TY5O5OI!$+..Qty7Q7Q3QK [W__T[11
.ty99	6s## @B	"$/ 	> 	>ByH$$W\\*bg>>
7=J33"'33327 CA"'AARYAAABBB  *k!:;;;;h&&7=J33"'33327 B@@@BI@@@AAA##K0000f$$$<RW<<	<<<===  	 	 ; FLL!Dc)nn!D!D!DEEE HLL!FS->->!F!F!FGGGDIIe,,7799:::""%#+t #     	 	 
 4.t{;;	6s##W__TY//
 	J.... QS"$/ 	> 	>ByJ&&W\\*bg>>
BGOOJ77$GGGG ECBGCCryCCCDDD>-"))2>:*FGGGG;A"NV"7"7bg"7"7"7rwK"));
*CDDDDh&&W\\*bg>>
 B@@@BI@@@AAA##J////f$$$<RW<<	<<<=== ~"" HF3~+>+>FFFGGG%%i@@@  	El 	EKKCC$5$5CCCDDD% 
	 
	Iw~~i(( 		)$$$33
**(((!#!8!8"   	 
**_4 4T
	 
	s   )3U
U*)U*c                 ,   |                                  }t          d| j         d| j                    t          d|d                     t          d|d                     t          d|d                     t          d	|d
                     dS )z!Print a summary of the sync plan.zSync plan: z -> z  Uploads: r   z  Downloads: r   z  Deletes: r   z	  Skips: r   N)r   r$  rF   r   )r   r   s     r'   _print_plan_summaryr8    s    llnnG	
4
4
4
4
4555	
,	*
,
,---	
0'+.
0
0111	
,	*
,
,---	
(gg&
(
()))))r)   )r   r   r   r   r   includeexcludefilter_fromr   applydry_runr  quiettokenr9  r:  r;  r<  r=  r>  r?  c                   |ddl m}  ||          }|r[| s|rt          d          |t          d          |rt          d          |rt          d          |rt          d	          |rt          d
          |	rt          d          |
rt          d          |rt          d          |rt          d          |rt          d          t          |          }t	          |           }|st          |           t          d           |rt                       	 t          ||||           |rt                       n# |rt                       w w xY w|st          d           |S | r|st          d          t          |           }t          |          }|r|rt          d          |s|st          d          |r|rt          d          |r|rt          d          |r|rt          d          |rQt          j                            |          r1t          j                            |          st          d|           n1t          j                            |           st          d|            d}|
rt          |
          }t!          ||	|          }t	          | o|           }t#          | |||||||||
  
        }|rt%          |t&          j                   |S |r5t+          ||           |s!t          |           t          d|            |S |st          |           |                                }|d         d k    r+|d!         d k    r|d"         d k    r|st          d#           |S |st          d$           |rt                       	 t          ||||           |rt                       n# |rt                       w w xY w|st          d           |S )%a  Sync files between a local directory and a bucket.

    This is equivalent to the ``hf buckets sync`` CLI command. One of ``source`` or ``dest`` must be a bucket path
    (``hf://buckets/...``) and the other must be a local directory path.

    Args:
        source (`str`, *optional*):
            Source path: local directory or ``hf://buckets/namespace/bucket_name(/prefix)``.
            Required unless using ``apply``.
        dest (`str`, *optional*):
            Destination path: local directory or ``hf://buckets/namespace/bucket_name(/prefix)``.
            Required unless using ``apply``.
        api ([`HfApi`]):
            The HfApi instance to use for API calls.
        delete (`bool`, *optional*, defaults to `False`):
            Delete destination files not present in source.
        ignore_times (`bool`, *optional*, defaults to `False`):
            Skip files only based on size, ignoring modification times.
        ignore_sizes (`bool`, *optional*, defaults to `False`):
            Skip files only based on modification times, ignoring sizes.
        existing (`bool`, *optional*, defaults to `False`):
            Skip creating new files on receiver (only update existing files).
        ignore_existing (`bool`, *optional*, defaults to `False`):
            Skip updating files that exist on receiver (only create new files).
        include (`list[str]`, *optional*):
            Include files matching patterns (fnmatch-style).
        exclude (`list[str]`, *optional*):
            Exclude files matching patterns (fnmatch-style).
        filter_from (`str`, *optional*):
            Path to a filter file with include/exclude rules.
        plan (`str`, *optional*):
            Save sync plan to this JSONL file instead of executing.
        apply (`str`, *optional*):
            Apply a previously saved plan file. When set, ``source`` and ``dest`` are not needed.
        dry_run (`bool`, *optional*, defaults to `False`):
            Print sync plan to stdout as JSONL without executing.
        verbose (`bool`, *optional*, defaults to `False`):
            Show detailed per-file operations.
        quiet (`bool`, *optional*, defaults to `False`):
            Suppress all output and progress bars.
        token (Union[bool, str, None], optional):
            A valid user access token. If not provided, the locally saved token will be used.

    Returns:
        [`SyncPlan`]: The computed (or loaded) sync plan.

    Raises:
        `ValueError`: If arguments are invalid (e.g., both paths are remote, conflicting options).

    Example:
        ```python
        >>> from huggingface_hub import HfApi
        >>> api = HfApi()

        # Upload local directory to bucket
        >>> api.sync_bucket("./data", "hf://buckets/username/my-bucket")

        # Download bucket to local directory
        >>> api.sync_bucket("hf://buckets/username/my-bucket", "./data")

        # Sync with delete and filtering
        >>> api.sync_bucket(
        ...     "./data",
        ...     "hf://buckets/username/my-bucket",
        ...     delete=True,
        ...     include=["*.safetensors"],
        ... )

        # Dry run: preview what would be synced
        >>> plan = api.sync_bucket("./data", "hf://buckets/username/my-bucket", dry_run=True)
        >>> plan.summary()
        {'uploads': 3, 'downloads': 0, 'deletes': 0, 'skips': 1, 'total_size': 4096}

        # Save plan for review, then apply
        >>> api.sync_bucket("./data", "hf://buckets/username/my-bucket", plan="sync-plan.jsonl")
        >>> api.sync_bucket(apply="sync-plan.jsonl")
        ```
    Nr   r   )r?  z,Cannot specify source/dest when using apply.z#Cannot specify both plan and apply.z'Cannot specify delete when using apply.z-Cannot specify ignore_times when using apply.z-Cannot specify ignore_sizes when using apply.z(Cannot specify include when using apply.z(Cannot specify exclude when using apply.z,Cannot specify filter_from when using apply.z)Cannot specify existing when using apply.z0Cannot specify ignore_existing when using apply.z(Cannot specify dry_run when using apply.)enabledzExecuting plan...)r  r   zSync completed.z7Both source and dest are required (unless using apply).z?Remote to remote sync is not supported. One path must be local.z?One of source or dest must be a bucket path (hf://buckets/...).z2Cannot specify both ignore_times and ignore_sizes.z1Cannot specify both existing and ignore_existing.z%Cannot specify both dry_run and plan.z!Destination must be a directory: z&Source must be an existing directory: r   )
rF   r   r   r   r   r   r   r   r   r   zPlan saved to: r   r   r   r   zNothing to sync.z
Syncing...)hf_apir   r#   r  r   r8  r$  r   r6  r   r|   rS   r   r*  r   r   r   r  r  sysstdoutr  r   )rF   r   r   r   r   r   r   r   r9  r:  r;  r   r<  r=  r  r>  r?  r   	sync_planr   source_is_bucketdest_is_bucketr   r   r   s                            r'   sync_bucket_internalrH    sm   F !!!!!!e%    ) 	MT 	MKLLLBCCC 	HFGGG 	NLMMM 	NLMMM 	IGHHH 	IGHHH 	MKLLL 	JHIII 	QOPPP 	IGHHHu%%	I... 	'	***%&&& 	$!###	')S'&IIII '$&&&  '$&&&&'  	%#$$$  T TRSSS&v..$T**N \N \Z[[[ \N \Z[[[ O OMNNN NO NLMMM B4 B@AAA  P7>>$ 	Id(;(; 	IGGGHHHw}}V$$ 	PNfNNOOO L 7)+66"  !  N E	 9'k:::F"!!'%  I  Isz*** 9d### 	,	****D**+++  'I&&&!!GyQ7;#71#<#<ASWXAXAX 	&$%%% l  #igfEEEE 	# """  	# """"	#  !   s   &E
 
E,O O#)FFFFFNN)FN)NN)Gr?   r   r  rP   rS   rC  rU   dataclassesr   r   r   r   pathlibr   typingr   r	   r
   r   r   r   r    r   r   errorsr   utilsr   r   r   r   utils._terminalr   rB  r   
get_loggerr<   r   rw   r   r@   r   r(   r+   rE   rY   r\   r_   rk   rq   rz   rB   r|   r~   r   r   r   r   rC   floatr   r   r   r   r  r  r  r  r6  r8  rH  r4   r)   r'   <module>rQ     s}   
       				 



  ( ( ( ( ( ( ( ( ' ' ' ' ' ' ' '       I I I I I I I I I I I I I I I I                 ' ' ' ' ' ' [ [ [ [ [ [ [ [ [ [ [ [ ' ' ' ' ' '   
	H	%	%   c eCHo     ' ' ' ' ' ' ' '@ 
 
 
 
 
 
 
 
*         $         %7 %7 %7 %7 %7 %7 %7 %7P > > > > > > > >, D D D D D D D D,IS IU38_ I I I I*# *$ * * * * 	- 	- 	- 	- 	- 	- 	- 	- 
 
 
 
 
 
 
 
8. . . . . . . .bC DsCx,A    :+# +(5c53I*J + + + +*2G 2 2S 2XeTWY\^cehThNiEj 2 2 2 2DPE Pc P P P P$ "&ES ES ES
ES ()ES 	ES
 ES ES ES ES ES ES ES ES #ES ES ES ES ESX !.2 c cc
c 
c 	c
 c c c c ]+c SMc c c c cV,h ,d , , , ,:X # $    ## #( # # # #V` ` `w ` `xX[} `hl ` ` ` `F*h *4 * * * *  !n
 !#'#'!%$(%n n nSMn
3-n 
	n
 n n n n n d3i n d3i n #n 3-n C=n n  !n" #n$ sD!%n& 'n n n n n nr)   