
    -:jq                     
   d Z ddlZddlZddlZddlZddlZddlmZmZm	Z	 ddl
mZ ddlmZ ddlmZmZmZmZmZ ddlmZ dd	lmZmZ dd
lmZmZmZmZmZ erddlm Z m!Z!  ejD                  e#      Z$dZ%g dZ&dZ'dZ(dZ)dZ* e+       Z,dZ-dZ.dZ/e-fde0de0de1de2fdZ3de0de2fdZ4 G d d      Z5 G d d      Z6 G d  d!      Z7 G d" d#      Z8dddd$dd%d&d'd(e2d)e2d*e9e   d+e9e   d,e2d-e2dz  d.e2e:z  dz  d/e2dz  d0e:d1e2dz  dd2fd3Z;y)4ac  Streamed, multi-commit upload of a folder on top of the Xet upload protocol.

How it works:

- The **coordinator** (caller's thread) walks the list of files and asks the Hub, 256 files at a
  time, what each file is (regular git blob, xet file, ignored). Regular files are accumulated
  directly; xet files are registered into a `XetSession` upload-commit, which chunks, deduplicates,
  retries and uploads them in the background while the coordinator keeps going. No Python-side
  sha256 computation: `hf_xet` computes it during chunking (single read pass over each file).
- Whenever enough files have accumulated (adaptive batch size), the batch is handed over to the
  **committer** thread which joins the xet uploads, drops unchanged files, and creates a git
  commit for the batch. While a batch is being committed, the coordinator is already uploading
  the next one.
- Interrupted uploads are resumable by simply re-running the same call: already-committed files
  are dropped (no-op detection against the remote oid) and already-uploaded chunks are
  deduplicated by the xet storage backend, transferring ~0 bytes.
    N)TYPE_CHECKINGAnyCallable)quote   )	constants)CommitOperationAddCommitOperationDelete_fetch_upload_modes_send_commit_warn_on_overwriting_operations)RepositoryNotFoundError)are_progress_bars_disabledlogging)XetTokenTypeabort_xet_sessionget_xet_sessionxet_connection_info_refresh_urlxet_headers_without_auth)
CommitInfoHfApi   )
   2   K   d   }         i  iX       g      D@g     r@i  @r         ?g      >@currenttotalwidthreturnc                 h    |dk  rd|z  S t        t        | |z  d      |z        }d|z  d||z
  z  z   S )Nr   u   ░      ?u   █)intmin)r#   r$   r%   filleds       b/var/www/html/tokenscope/api/venv/lib/python3.12/site-packages/huggingface_hub/_upload_pipeline.py_barr-   U   sE    zu}Wu_c*U23F6>EUV^444    nc                     dD ]:  }t        |       dk  r%| dk  r	| d| c S | dk  r	| d| c S | d| c S | dz  } < | ddS )	N)BkBMBGBTBr    
   z.2fr   .1fz.0fPB)abs)r/   units     r,   _format_bytesr;   \   sv    - q6D=2vC''SC''WTF##	T	 WB<r.   c                       e Zd ZdZdZddededdfdZddZdd	Z	d
eddfdZ
d
eddfdZd
eddfdZdee   ddfdZd
eddfdZdeddfdZddZddZddZddZdefdZdefdZdefdZdefdZy)_LiveDisplayuR  Three-line live progress display on stderr::

        Preparing   ████████████████████  11,100 / 11,100 ✓
        Uploading   ██████████████░░░░░░  580 / 603 files  3.8GB · 19.7MB/s
        Committing  ██████████████████░░  10,800 / 11,100  14 commits

    A small renderer thread redraws the three lines in-place every ~0.5 s on a TTY
    (worker threads only update counters under a lock). When stderr is not a TTY,
    it falls back to a periodic ``logger.info`` summary instead.

    Disabling progress bars (e.g. agent output mode) only turns off the TTY renderer:
    the non-TTY log summaries are gated by the logger verbosity alone, so consumers
    tailing stderr during a long upload still see periodic progress.
       total_filesenabledr&   Nc                    || _         |xr t        j                  j                         | _        | j                  xs# t
        j                  t        j                        | _	        t        j                         | _        d| _        t        j                         | _        d | _        d| _        d| _        d| _        t)               | _        d| _        d| _        d| _        d| _        d| _        d | _        y )NFr           )_totalsysstderrisatty_ttyloggerisEnabledForr   INFO_active	threadingLock_lock_drawnEvent_stop_event_thread	_prepared_ignored
_xet_totalset	_xet_done
_committed_nb_commits
_xet_bytes
_speed_ema_prev_bytes
_prev_time)selfr?   r@   s      r,   __init__z_LiveDisplay.__init__z   s    !3

 1 1 3	yyEF$7$7$E^^%
$??,04 #&5 (,r.   c                    | j                   sy | j                  rMt        j                  j	                  d| j
                  dd       t        j                  j                          n$t        j                  d| j
                  dd       t        j                  | j                  dd      | _        | j                  j                          y )NzFound ,z files to upload
z files to uploadzhf-upload-displayTtargetnamedaemon)rK   rG   rD   rE   writerC   flushrH   inforL   Thread_render_looprR   startr^   s    r,   rk   z_LiveDisplay.start   s    ||99JJvdkk!_4FGHJJKK&Q/?@A ''t/@/@GZcghr.   c                     | j                   4| j                  j                          | j                   j                          | j                  r&| j
                  5  | j                          d d d        y y # 1 sw Y   y xY wN)rR   rQ   rV   joinrG   rN   _redrawrl   s    r,   closez_LiveDisplay.close   sd    <<#  "LL99    s   A44A=r/   c                 p    | j                   5  | xj                  |z  c_        d d d        y # 1 sw Y   y xY wrn   )rN   rS   r^   r/   s     r,   notify_preparedz_LiveDisplay.notify_prepared   s,    ZZ 	 NNaN	  	  	    ,5c                 p    | j                   5  | xj                  |z  c_        d d d        y # 1 sw Y   y xY wrn   )rN   rT   rs   s     r,   notify_ignoredz_LiveDisplay.notify_ignored   s,    ZZ 	MMQM	 	 	ru   c                 p    | j                   5  | xj                  |z  c_        d d d        y # 1 sw Y   y xY wrn   )rN   rU   rs   s     r,   notify_xet_registeredz"_LiveDisplay.notify_xet_registered   ,    ZZ 	!OOq O	! 	! 	!ru   namesc                 |    | j                   5  | j                  j                  |       d d d        y # 1 sw Y   y xY wrn   )rN   rW   update)r^   r{   s     r,   notify_xet_uploadedz _LiveDisplay.notify_xet_uploaded   s0    ZZ 	)NN!!%(	) 	) 	)s   2;c                 p    | j                   5  | xj                  |z  c_        d d d        y # 1 sw Y   y xY wrn   )rN   rX   rs   s     r,   notify_skippedz_LiveDisplay.notify_skipped   rz   ru   n_filesc                     | j                   5  | xj                  |z  c_        | xj                  dz  c_        d d d        y # 1 sw Y   y xY w)Nr   )rN   rX   rY   )r^   r   s     r,   notify_commitz_LiveDisplay.notify_commit   s=    ZZ 	"OOw&O!	" 	" 	"s   +AA
c                 R      j                   syddt        dt        ddf fd}|S )aS  Progress callback for one ``new_upload_commit``.

        The byte counters in ``group_report`` are cumulative *per upload-commit* and several
        upload-commits can be in flight at once (one finalizing, one filling), so each commit
        gets its own closure tracking its own previous value; increments are summed globally.
        Nr   group_reportitem_reportsr&   c                 r   j                   5  | j                  }xj                  t        d|z
        z  c_        ||j	                         D ]Q  }|j
                  dkD  s|j                  |j
                  k(  s-j                  j                  |j                         S 	 d d d        y # 1 sw Y   y xY wNr   )
rN   total_transfer_bytes_completedrZ   maxvaluestotal_bytesbytes_completedrW   add	item_name)r   r   	completeditemprevr^   s       r,   callbackz/_LiveDisplay.new_xet_callback.<locals>.callback   s     ;(GG	3q)d*:#;; (//1 ;D''!+0D0DHXHX0X**4>>:;	; ; ;s   AB-"B-<'B--B6)rK   r   )r^   r   r   s   ` @r,   new_xet_callbackz_LiveDisplay.new_xet_callback   s5     ||	;3 	;c 	;d 	; r.   c                    d}| j                   j                  t              s| j                  5  | j	                          | j
                  r| j                          nUt        j                         |z
  t        k\  r7t        j                  | j                                t        j                         }d d d        | j                   j                  t              sy y # 1 sw Y   +xY w)NrB   )rQ   wait_REFRESH_INTERVALrN   _update_speedrG   rp   time	monotonic_NON_TTY_LOG_INTERVALrH   rh   _summary)r^   last_logs     r,   rj   z_LiveDisplay._render_loop   s    ""''(9: 0""$99LLN^^%04IIKK0#~~/H0 ""''(9:0 0s   BCC$c                 :   t        j                         }| j                  c|| j                  kD  rT| j                  | j                  z
  || j                  z
  z  }| j
                  dk(  r|nd|z  d| j
                  z  z   | _        || _        | j                  | _        y )Nr   g333333?gffffff?)r   r   r]   rZ   r\   r[   )r^   nowrates      r,   r   z_LiveDisplay._update_speed   s    nn??&3+@OOd&6&663;PQD&*oo&:dd
SSWSbSbMb@bDO??r.   c                    | j                   r-t        j                  j                  d| j                   d       t        j                         j                  }| j                         | j                         | j                         fD ]C  }t        |      |dz
  kD  r|d |dz
   dz   n|}t        j                  j                  d| d       E t        j                  j                          d| _         y )	Nz[Ar      z...z[K
T)rO   rD   rE   rf   _N_LINESshutilget_terminal_sizecolumns_line_preparing_line_uploading_line_committinglenrg   )r^   r%   line	truncateds       r,   rp   z_LiveDisplay._redraw   s    ;;JJuT]]O156((*22))+T-A-A-CTEZEZE\] 	7D58Y5J[uqy)E1PTIJJx	{"56	7 	

r.   c                     | j                   | j                  k\  rdnd}dt        | j                   | j                         d| j                   dd| j                  d| S )N    ✓ z  Preparing     ra    / )rS   rC   r-   )r^   dones     r,   r   z_LiveDisplay._line_preparing   s[    4;;6vBT^^T[[ AB"T^^TUDVVYZ^ZeZefgYhimhnoor.   c           	      v   | j                   dk(  r7| j                  | j                  k\  rt        dd      nt        dd      }d| dS t	        | j
                        }g }| j                  dkD  r$|j                  t        | j                               | j                  dkD  r'|j                  t        | j                         d       |rddj                  |       nd}| j                  | j                  k\  r|| j                   k\  rd	nd}dt        || j                          d|d
d| j                   d
d| | 	S )Nr   r   z  Uploading   z  -z/sr   u    · r   r   ra   r   z files)rU   rS   rC   r-   r   rW   rZ   appendr;   r[   ro   )r^   barn_donepartsextrar   s         r,   r   z_LiveDisplay._line_uploading   s   ??a $$++ =$q!*41:C#C5,,T^^$??QLLt78??QLLM$//:;2>?-2"V[['()4;;66T__;TvZ\VT__ =>b
#doo^_M``fglfmnrmsttr.   c                    | j                   | j                  z
  }| j                  dkD  rd| j                   dnd}| j                  |cxk\  rdkD  rn ndnd}dt	        | j                  |       d| j                  dd	|d| | S )
Nr   r   z commitsr   r   r   z  Committing  ra   r   )rC   rT   rY   rX   r-   )r^   	effectivecommits_strr   s       r,   r   z_LiveDisplay._line_committing	  s    KK$--/	9=9I9IA9M4++,H5SUI99vrT$//9=>bQR@SSVW`abVcdocpqupvw	
r.   c                     d| j                   dd| j                  ddt        | j                        dd| j                  ddt        | j                         d| j                  dd| j                   dS )	NzUploading... ra   /z files checked, z uploaded (z transferred), z committed in z
 commit(s))	rS   rC   r   rW   rU   r;   rZ   rX   rY   rl   s    r,   r   z_LiveDisplay._summary  sy    DNN1-Qt{{1o=M4>>"1%Qtq&9]SWSbSbEcDddsq!0@0@/AM	
r.   )Tr&   N)r&   zCallable | None)__name__
__module____qualname____doc__r   r)   boolr_   rk   rq   rt   rw   ry   liststrr~   r   r   r   rj   r   rp   r   r   r   r    r.   r,   r=   r=   h   s     H-C -$ -$ -2	       !s !t !)c )t )! ! !"S "T "
2	0+p pu u
# 

# 
r.   r=   c                   J    e Zd ZdZd
dZedefd       ZdededdfdZ	d
d	Z
y)_CommitPacerzOAdaptive number of files per commit, to stay below server-side commit timeouts.r&   Nc                     t         | _        y rn   )INITIAL_COMMIT_SIZE_INDEX_indexrl   s    r,   r_   z_CommitPacer.__init__  s	    /r.   c                 (    t         | j                     S rn   )COMMIT_SIZE_SCALEr   rl   s    r,   rc   z_CommitPacer.target  s     --r.   durationnb_filesc                     |t         k  r>|| j                  k\  r/t        | j                  dz   t	        t
              dz
        | _        y |t         kD  rt        | j                  dz
  d      | _        y y Nr   r   )TARGET_COMMIT_DURATIONrc   r*   r   r   r   r   )r^   r   r   s      r,   record_successz_CommitPacer.record_success#  s[    ,,T[[1HdkkAos3D/E/IJDK..dkkAoq1DK /r.   c                 @    t        | j                  dz
  d      | _        y r   )r   r   rl   s    r,   record_failurez_CommitPacer.record_failure)  s    $++/1-r.   r   )r   r   r   r   r_   propertyr)   rc   floatr   r   r   r.   r,   r   r     sD    Y0 . . .2u 2 2 2.r.   r   c                       e Zd ZdZddZy)_BatchzSA group of files destined to a single git commit, with their in-flight xet uploads.Nc                 n    g | _         d| _        d | _        g | _        t	        j
                         | _        y r   )opsregular_bytes
xet_commithandlesr   r   
created_atrl   s    r,   r_   z_Batch.__init__0  s-    -/"##=?!%!1r.   r   )r   r   r   r   r_   r   r.   r,   r   r   -  s
    ]2r.   r   c                       e Zd Zdddededee   dee   deded	z  d
eez  d	z  ded	z  deded	z  dd	fdZddZ	ddZ
dedd	fdZdedd	fdZddZdedd	fdZdee   dd	fdZdee   dd	fdZddZy	)_UploadPipelineapir   repo_id	repo_typeadd_operationsdelete_operationscommit_messagecommit_descriptionNtokenrevision	create_prparent_commitr&   c       
            || _         || _        || _        || _        || _        || _        || _        || _        |j                  |      | _	        |	xs t        j                  | _        |
| _        || _        t        | j                  d      | _        | j                   | _        d | _        d | _        d| _        d | _        t-               | _        t1        j2                  d      | _        g | _        t9        j:                         | _        t?        tA        |      tC                      | _"        tG        tH        jJ                  ||| j                   |jL                        }|
r|d	z  }tO               | _(        || j                  tS        | j                        d
| _*        d | _+        |D ]O  }|jX                  dk(  s|j[                         5 }|j]                         j_                         | _+        d d d         y  y # 1 sw Y    y xY w)N)r   r   safer   r   )maxsize)r?   r@   )
token_typer   r   r   endpointz?create_pr=1)token_refresh_urltoken_refresh_headerscustom_headersz
.gitignore)0r   r   r   r   r   r   r   r   _build_hf_headersheadersr   DEFAULT_REVISIONr   r   r   r   base_revision_quotedcommit_revision_quotedpr_urlpr_revision
nb_commitslast_commit_infor   pacerqueueQueuebatch_queueerrorsrL   rP   abort_eventr=   r   r   displayr   r   WRITEr   r   xet_sessionr   xet_commit_kwargsgitignore_contentpath_in_repoas_filereaddecode)r^   r   r   r   r   r   r   r   r   r   r   r   refresh_urlopfs                  r,   r_   z_UploadPipeline.__init__9  s    ",!2,"4
,,5,9 >I$>$>"*
 %*$--b$A! '+&?&?#"&'+59!^
 ).A(>+-$??,#N0CQkQmMmn 6#))..\\
 >)K*,!,%)\\6t||D"
 .2  	B,.ZZ\ ?Q-.VVX__->D*?		?s   
$G::H	c           	         t        g | j                  | j                         t        j                  | j
                  dd      }|j                          | j                  j                          	 | j                          	 | j                  j                         r|j                  d       n/| j                  j!                  t"               |j                          | j                  j%                          | j                  j                         r=| j&                  1t(        j+                  d| j,                   d| j&                   d       	 | j.                  r| j.                  d	   | j1                         S # t        $ r& | j                  j                          t                 w xY w# | j                  j                         r|j                  d       n/| j                  j!                  t"               |j                          | j                  j%                          | j                  j                         r>| j&                  1t(        j+                  d| j,                   d| j&                   d       w w w xY w)
Nzhf-upload-committerTrb   r6   timeoutzUpload to pull request zF did not complete. To resume into the same PR, re-run with `revision="z^"` (without `create_pr=True`). Re-running with `create_pr=True` would open a new pull request.r   )r   r   r   rL   ri   _committer_looprk   r  _coordinator_loopBaseExceptionr  rV   r   is_setro   r  put	_SENTINELrq   r   rH   warningr   r  _final_commit_info)r^   	committers     r,   runz_UploadPipeline.run  s   '(W$*@*@(W4CVCV(WX$$D,@,@G\eij		""$ &&( r*  $$Y/ LL &&(T-=-=-I-dkk] ;88<8H8H7I JLL
 ;;++a. &&((-  	  "	
 &&( r*  $$Y/ LL &&(T-=-=-I-dkk] ;88<8H8H7I JLL .J(s   1E: :/F))F, ,CI<c                    dd l }t               }t        dt        | j                        t
              D ]a  }| j                  j                         r| j                  |        y | j                  ||t
        z    }	 t        || j                  | j                  | j                  | j                  | j                  j                  | j                   | j"                         | j,                  j/                  t        |             |D ]  }|j0                  r?t2        j5                  d|j6                   d       | j,                  j9                  d       O|j:                  dk(  r+|xj<                  |j>                  j@                  z  c_        nL|jB                  E | jD                  jF                  dd| j,                  jI                         i| jJ                  |_!        |j>                  jL                  r$|j>                  jN                  jQ                         n|jR                  }tU        |jV                  tX              r3|jB                  j[                  |jV                  ||j6                  	      }	n0|jB                  j]                  t_        |jV                        |
      }	|j`                  jc                  ||	f       | j,                  je                  d       |jf                  jc                  |       t        |jf                        | jh                  jj                  k\  sW|j<                  tl        k\  sDto        jp                         |jr                  z
  tt        kD  sYt        |jf                        dkD  ss| jw                  |       t               } d | jw                  |       y # t$        $ r}ddlm} |j+                  |        d }~ww xY w)Nr   )	additionsr   r   r   r   r   r   r  r   )$_CREATE_COMMIT_NO_REPO_ERROR_MESSAGEzSkipping upload for 'z' (ignored by gitignore rules).regularprogress_callback)sha256rd   )r%  r   )<hf_xetr   ranger   r   PREUPLOAD_BATCH_SIZEr  r  _abort_batchr   r   r   r   r   r   r   r   r  r   hf_apir"  append_to_messager  rt   _should_ignorerH   debugr  rw   _upload_moder   upload_infosizer   r	  new_upload_commitr   r
  	is_hashedr%  hexCOMPUTE_SHA256
isinstancepath_or_fileobjbytesstart_upload_bytesstart_upload_filer   r   r   ry   r   r  rc   REGULAR_CONTENT_BYTES_BUDGETr   r   r   MAX_COMMIT_INTERVAL_enqueue)
r^   r&  batchrk   chunker"  r  sha_arghandles
             r,   r  z!_UploadPipeline._coordinator_loop  s   1c$"5"568LM 5	%E&&(!!%(''8L0LME##"nn LL LL!66!XX.."nn&*&<&<	 LL((U4 %$$LL#88IIh!ijLL//2??i/''2>>+>+>>'''/+M4+;+;+M+M ,.2ll.K.K.M,QUQgQg,(
 >@^^=U=Ubnn33779[a[p[pG!""4"4e<!&!1!1!D!D..wR__ "E " "'!1!1!C!CCHZHZD[dk!C!lMM(("f6LL66q9		  $ 		Ndjj&7&77**.JJ(5+;+;;>QQVYZ_ZcZcVdghVhMM%("HE?%-5	%l 	eM + H##$HI	s   ;A#N**	O3OOr=  c                 J   t        |j                        dk(  r(| j                  dk(  rt        | j                        dkD  sy | j                  j                         s	 | j                  j                  |d       y | j                  |       y # t        j                  $ r Y `w xY w)Nr   r(   r  )r   r   r   r   r  r  r  r  r  Fullr)  r^   r=  s     r,   r<  z_UploadPipeline._enqueue  s    uyy>Q1(<TE[E[A\_`A`""))+  $$UC$8 	%  :: s   B B"!B"c                 r    |j                   	 |j                   j                          y y # t        $ r Y y w xY wrn   )r   abort	ExceptionrD  s     r,   r)  z_UploadPipeline._abort_batch  s>    '  &&( (  s   * 	66c                    	 	 | j                   j                  d      }|t        u ry 	 | j                  j                         s| j                  |       n| j                  |       e# t        j                  $ r | j                  j                         rY y Y w xY w# t        $ rP}| j                  |       | j                  j                  |       | j                  j                          Y d }~d }~ww xY w)Nr"   r  )r  getr  Emptyr  r  r  _process_batchr)  r  r  r   rV   )r^   r=  r?  s      r,   r  z_UploadPipeline._committer_loop  s    ((,,S,9
 	!'''..0''.%%e,  ;; ##**, ! '!!%(""1%  $$&&'s)   A' =B '-BB	C5%AC00C5c           
         |j                   |j                   j                          |j                  D ]h  \  }}|j                  j                  sFt
        j                  |j                         j                  j                        |j                  _	        d|_
        j | j                  j                  |j                  D cg c]@  \  }}t        |j                  t
              st        |j                        n|j                   B c}}       g }|j"                  D ]w  }|j$                  X|j$                  |j&                  k(  r?t(        j+                  d|j                    d       | j                  j-                  d       g|j/                  |       y t1        |      dkD  s'| j2                  dk(  r+t1        | j4                        dkD  r| j7                  |       y y y c c}}w )NTzSkipping commit for 'z' (file unchanged).r   r   )r   wait_to_finishr   r/  r2  r7  fromhexresultxet_infor%  _is_uploadedr  r~   r5  r6  r   r  r   _remote_oid
_local_oidrH   r-  r   r   r   r   r   _commit_with_split)r^   r=  r  rA  _ops_to_commits         r,   rK  z_UploadPipeline._process_batch  s    '++-#mm '
F~~//,1MM&--/:R:R:Y:Y,ZBNN)"&'
 LL,, "'A 4>b>P>PRW3XC**+^`^m^mm )) 	%B~~)bnn.M4R__4EEXYZ++A.  $	% }!doo&:s4CYCY?Z]^?^##M2 @_&:#s   AG!
r   c                    	 | j                  |       y # t        $ r}| j                  j                          t	        |      t
        d   k  r t        j                  dt	        |       d|d       | j                  j                  }t        dt	        |      |      D ]  }| j                  ||||z            Y d }~y d }~ww xY w)Nr   z
Commit of z files failed (z). Retrying in smaller chunks.)
_do_commitrG  r  r   r   r   rH   r  rc   r'  rT  )r^   r   r?  rc   rk   s        r,   rT  z"_UploadPipeline._commit_with_split'  s    		EOOC  	EJJ%%'3x,Q//NNZCzEcdeZZ&&Fq#c(F3 E''EEFN(CDE	Es    	CBB??Cc                     | j                   r| j                  | j                  j                  | j                  | j
                  | j                  | j                  | j                        }|j                  t        d      |j                  | _        |j                  | _        t        |j                  d      | _        t        |      }| j                   dk(  rt        | j"                        |z   }| j                   dk(  r| j
                  n| j
                   d| j                   dz    d}t%        j&                         }t)        |i || j                  xs d| j                  | j                  | j*                  | j                  | j                  j,                  | j                   dk(  r| j.                  nd d	
      | _        t%        j&                         |z
  }| j2                  j5                  |t7        |             | xj                   dz  c_        |D ]	  }d	|_         | j:                  j=                  t7        |             t>        jA                  dt7        |       d|dd| j0                  jB                          y )N)r   titler   descriptionr   zCServer did not return a git reference for the created pull request.r   r   r   z (part r   )T)
operationsfiles_to_copyr   r   r   r   r   r   r   r   retry_on_errorz
Committed z file(s) in r7   zs: )"r   r   r   create_pull_requestr   r   r   r   r   git_reference
ValueErrorurlr   r   r   r   r   r   r   r   r   r   r   r   r   r  r   r   _is_committedr  r   rH   r-  
commit_url)r^   r   prr]  r   t0r   r  s           r,   rX  z_UploadPipeline._do_commit3  s   >>d..6 --))jj 33.. . B ' !fgg&&DK!//D*/0@0@r*JD' $S	
??ad445
BJ $(??a#7D@S@S?TT[\`\k\kno\o[ppq=r 	 ^^ !-!)#66<"nnLLLL00XX&&0410D$,,$!
 >>#b(

!!(CH51 	$B#B	$""3s8,z#c(<~SI^I^IiIiHjklr.   c                    ddl m} | j                  t        j	                  d       | j
                  j                  | j                  | j                  | j                        }| j
                  j                  }| j                  t        j                  k7  r| d| j                   d} || d| j                   d|j                   | j                  | j                  xs d|j                  | j
                  j                  	      S | j                   dkD  r#t        j#                  d
| j                    d       | j$                  | || j                  j&                  | j                  j                  | j                  j                  | j                  j(                  | j$                  | j
                  j                        S | j                  S )Nr   )r   zPNo files have been modified since last commit. Skipping to prevent empty commit.)r   r   r   r   sz/commit/r   )re  r   r   oid	_endpointzUpload completed in z	 commits.)re  r   r   rj  r   rk  )r*  r   r   rH   r  r   	repo_infor   r   r   r   r   REPO_TYPE_MODELshar   r   r   rh   r   re  rj  )r^   r   rh   
url_prefixs       r,   r  z"_UploadPipeline._final_commit_infoj  st   &  ( NNmn88%%dlldnn_c_l_l%mD**J~~!:!:: *|1T^^,<A>
(\4<<.
K#22#'#:#:#@bHH((++  ??QKK.t.?yIJ;;"00;;#44CC#'#8#8#K#K))--{{((++  $$$r.   )r&   r   r   )r   r   r   r   r   r	   r
   r   r_   r  r  r   r<  r)  r  rK  rT  rX  r  r   r.   r,   r   r   8  s4   GG 	G
 G /0G   56G G  $JG TzD G *G G TzG 
GR)B:x
!f 
! 
!& T '(3F 3t 3>
Ed+=&> 
E4 
E3md#56 3m4 3mn%r.   r   F)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   c       
         J    t        | |||||||||	|
      j                         S )zUpload a prepared list of operations through the streamed multi-commit pipeline.

    Requires `hf_xet` to be installed. See module docstring for the architecture.
    )
r   r   r   r   r   r   r   r   r   r   )r   r  )r   r   r   r   r   r   r   r   r   r   r   s              r,   pipelined_uploadrq    s;    & %+%-# 
cer.   )<r   r  r   rD   rL   r   typingr   r   r   urllib.parser   r   r   _commit_apir	   r
   r   r   r   r  r   utilsr   r   
utils._xetr   r   r   r   r   r*  r   r   
get_loggerr   rH   r(  r   r   r   r;  r:  objectr  
_BAR_WIDTHr   r   r   r)   r   r-   r;   r=   r   r   r   r   r   rq  r   r.   r,   <module>rz     s  $   
   / /    , 6  )			H	%   E      1 H	 
   5? 5% 5 5c 53 5	U 	s 	n
 n
b. .(2 2P% P%v
 &*# $	  	
 +, 12  d
 : Dj  : r.   