-:j@dZddlZddlZddlmZddlmZddlmZddl m Z m Z m Z ddl mZddlmZd d lmZmZmZmZmZmZd d lmZd d lmZmZe rd d lmZej>e Z!ejDdZ#dZ$dddZ%GddZ&e d8dee&de'dzde'de'de'dzde'dzde(e'e'fdzde)e'dzde*e)e(e)e(e'dzffdZ+Gdde Z,Gd d!e Z- d9d"d#d$e(de'dzde(e'e'fdzde'dzddf d%Z.d&e(fd'Z/d(e(fd)Z0d*e(fd+Z1d"d#d,e'ddfd-Z2d"d#d.e(d/e3d,e'ddf d0Z4d.e(d1e&d/e3de)e'fd2Z5d3e)e(d4e'de-fd5Z6d"d#d6e)e'd/e3de)e(fd7Z7y):z.Git LFS related type definitions and utilitiesN)Iterable)ceil)getsize) TYPE_CHECKINGBinaryIO TypedDict)unquote) constants)build_hf_headersfix_hf_endpoint_in_urlhf_raise_for_status http_backoffloggingvalidate_hf_hub_args) SliceFileObj)sha256 sha_fileobj)CommitOperationAddz^[0-9a-f]{40}$zlfs-multipart-uploadzapplication/vnd.git-lfs+json)Acceptz Content-Typec eZdZdZ ddedededzdedzfdZedefd Z e jd eddfd Z ede fd Z defd Z edefdZedefdZedefdZy) UploadInfoa Data structure holding required information to determine whether a blob should be uploaded to the hub using the LFS protocol or the regular protocol. The SHA256 of the blob is computed lazily: creating an `UploadInfo` from a local path only reads the first 512 bytes of the file. The full file is read (and hashed) only if `sha256` is accessed before it has been set. When a file is uploaded through the Xet protocol, the SHA256 is computed during upload (single read pass) and set afterwards. Args: size (`int`): Size in bytes of the blob sample (`bytes`): First 512 bytes of the blob sha256 (`bytes`, *optional*): SHA256 hash of the blob, if already known. Otherwise computed lazily from `source_path`. source_path (`str`, *optional*): Path to the local file the blob comes from. Required to lazily compute `sha256` if not provided. Nsizesampler source_pathcZ| | td||_||_||_||_y)Nz2Either `sha256` or `source_path` must be provided.) ValueErrorrr_sha256 _source_path)selfrrrrs U/var/www/html/tokenscope/api/venv/lib/python3.12/site-packages/huggingface_hub/lfs.py__init__zUploadInfo.__init__Js7 >k1QR R   'returnc|jI|jJt|jd5}t||_ddd|jS|jS#1swY|jSxYw)zZSHA256 of the blob. If not set yet, reads the whole file from `source_path` to compute it.Nrb)rropenr)r files r!rzUploadInfo.sha256Xsk << $$0 00d''. 1$*40  1||t|| 1||s A""A6valuec||_y)Nr)r r)s r!rzUploadInfo.sha256as  r#c|jduS)zVWhether the SHA256 is already known (accessing `sha256` will not trigger a file read).Nr+)r s r! is_hashedzUploadInfo.is_hashedes||4''r#cx|j|jjnd}d|jd|dS)NzzUploadInfo(size=z , sha256=))rhexr)r shas r!__repr__zUploadInfo.__repr__js9$(LL$> * *s AA datacbt|j}|t||dd|S)Nr5)rrr)rdigestlen)r7r9r1s r! from_byteszUploadInfo.from_bytesus-Tl!!#D $t*SAAr#fileobjc|jd}|jdtjt |}|j }|jdtj||||S)Nr5r)rrr)readseekioSEEK_SETrtell)r7r>rr1rs r! from_fileobjzUploadInfo.from_fileobjzsXc" Q $'"||~ Q $S88r#)NN)__name__ __module__ __qualname____doc__intbytesstrr"propertyrsetterboolr-r2 classmethodr8r=rrEr#r!rr5s0 $"& ( ( ( ( 4Z ( ]]Ed(4((=#=?S?? BeBB9899r#r upload_infostoken repo_typerepo_idrevisionendpointheaders transfersr$c||ntj}d}|tjvrtj|}|d||d} d||nddg|D cgc])} | jj | j d+c} d d } |d t |i| d <itt| |xsi}td| || } t| | j} | jdd}t|ts td| jd}t|t r|nd}|Dcgc]}d|vst#|c}|Dcgc]}d|vst%|c}|fScc} wcc}wcc}w)a Requests the LFS batch endpoint to retrieve upload instructions Learn more: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md Args: upload_infos (`Iterable` of `UploadInfo`): `UploadInfo` for the files that are being uploaded, typically obtained from `CommitOperationAdd.upload_info` token (`str` or `None`): An authentication token (see https://huggingface.co/settings/token). Pass `None` to fall back to the local cached token (or no token if unauthenticated). repo_type (`str`): Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`. repo_id (`str`): A namespace (user or an organization) and a repo name separated by a `/`. revision (`str`, *optional*): The git revision to upload to. endpoint (`str`, *optional*): The Hub endpoint to send the request to. Defaults to the value of `HF_ENDPOINT`. headers (`dict`, *optional*): Additional headers to include in the request transfers (`list`, *optional*): List of transfer methods to use. Defaults to ["basic", "multipart"]. Returns: `LfsBatchInfo`: 3-tuple: - First element is the list of upload instructions from the server - Second element is a list of errors, if any - Third element is the chosen transfer adapter if provided by the server (e.g. "basic", "multipart", "xet") Raises: [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) If an argument is invalid or the server response is malformed. [`HfHubHTTPError`] If the server returned an error. N/z.git/info/lfs/objects/batchuploadbasic multipartoidrr) operationrYobjects hash_algonameref)rSPOSTrXjsonrczMalformed response from servertransfererror)r ENDPOINTREPO_TYPES_URL_PREFIXESrr0rr LFS_HEADERSr rrriget isinstancelistrrL_validate_batch_actions_validate_batch_error)rRrSrTrUrVrWrXrY url_prefix batch_urlr]payloadresp batch_inforcchosen_transferobjs r!post_lfs_batch_infor{sb$/xY5G5GHJI55566yA *Aj\'2MNI"+"7Yg{=S '  }}((*    G '("34   ' =bG  7 IDJnnY-G gt $9:: nnZ0O)3OS)IotO29O#G3(>'??mno%Y/9M'$Y/33H=M ]+  x ,FL)J' f(=QJ ZJ Yv*akliJG ]++M&,A8L "  $5'B"..5599;YEZEZE_E_`  K( LLI**++>?@'I& fgqfrrtu  s - E22F lfs_actionct|jdtr0|jd*t|jdts t d|S).validates response from the LFS batch endpointrrz"lfs_action is improperly formatted)rprorLdictr)rs r!rr9sK :>>&)3/ ^^H % -JNN8> r#lfs_batch_actionscPt|jdtrt|jdts t d|jdijd}|jdijd}| t || t ||S)rrarz)lfs_batch_actions is improperly formattedrr]r)rprorLrJrr)rrrs r!rrrrCs (,,U3S 9jIZI^I^_eIfhk>lDEE%)))R8<t|jdtrt|jdts t d|S)rrarz'lfs_batch_error is improperly formattedrkmessagecode)rprorLrJrr)r error_infos r!rsrsQs **513 7JGZGZ[aGbdgresponses r!rr_sB   T  *&gz@H%&&&s6?rrc t||j|}t|||}td|t ||jj j t}t|y)z@ Uploads file using HF multipart LFS transfer protocol. )rrr)rbsorted_parts_urlsrrg)rirXN) _get_sorted_parts_urlsr_upload_parts_iterativelyr_get_completion_payloadrr0rnr)rbrrrrresponse_headerscompletion_ress r!rrssn /f)J_J_lvw1/@Z " $%5y7L7L7S7S7W7W7Y Z N 'r#rc Vt|jDcgc]1\}}|jrt|dkDrt |d|f3c}}dDcgc]\}}| }}}t|}|t |j |z k7r td|Scc}}wcc}}w)Nr c |dS)NrrQ)ts r!z(_get_sorted_parts_urls..s !A$r#)keyz0Invalid server response to upload large LFS file)sorteditemsisdigitr<rJrrr)rrrpart_numr_sorted_part_upload_urls num_partss r!rrs$-3LLN (Hj##%#h-!*;Xr"J/      Az   +,ID))J677KLL ""  s 6B  B%rracg}t|D]H\}}|jd}||dk(rtd|d|dz|j|dz|dJ||dS)Nrr[zInvalid etag (`z`) returned for part r )r~r)rar) enumeraterorappend)rrar part_numberrrs r!rrs "E()9:   Vzz&! <42:tf4I+XY/IZ[\ \ )Ao    ''r#rc6g}|jd5}t|D]S\}}t|||z|5}td||}t ||j |j dddU ddd|S#1swYjxYw#1swY|SxYw)NTr) seek_from read_limitrr)rrrrrrrX) rbrrrXr>part_idxpart_upload_url fileobj_slicepart_upload_ress r!rrsG   T  * 8g)23D)E 8 %Ho$x/% 8".uoM"Z#O4667 8 8 8 8 N 8 8 8 Ns"#B5B- BB BB)NNNN)NNN)8rIrBrecollections.abcrmathros.pathrtypingrrr urllib.parser huggingface_hubr utilsr r rrrr utils._lfsr utils.sharr _commit_apir get_loggerrFrcompile OID_REGEXLFS_MULTIPART_UPLOAD_COMMANDrnrrLrrqtupler{r}rrrrrrsrrJrrrrrQr#r!rs5 $55 %%*/   H % BJJ( ) 5-2 L9L9^  %)"&X:&X :XX X Dj X Dj X#s(^d "XCy4X 4:tDz3: -.XXv9 %) JA#JAJA :JA#s(^d " JA Dj JA  JAZT t  4 &S&T&(("6((RU(cf(ko(.#4#j#c#VZ[^V_#$ (d4j (s (GY (#8