
    -:j              *          d Z ddlZddlZddlZddlZddlmZmZ ddlmZ ddl	m
Z
mZ ddlmZmZmZ ddlmZ ddlZddlmZmZmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddl m!Z!m"Z" ddl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1 ddl2m3Z3m4Z4  ejj                  e6      Z7de8de8dz  de9e8e8dz  f   fdZ:dZ;ee8 ejx                  d      f   Z=ee8dz   ej|                  d      f   Z?ee8dz   ej|                  d e(e            f   Z@eeAe8   dz   ej|                  ddd      f   ZBee8dz   ej|                  d      f   ZCeeD ej|                  dd d!      f   ZEee8dz   ej|                  d"      f   ZFeeAeG   dz   ej|                  d#d$      f   ZHeeD ej|                  d%d&      f   ZIeeAe8   dz   ej|                  d'd(      f   ZJee8dz   ej|                  d)d*d+      f   ZKeeDdz   ej|                  d,      f   ZLeeDdz   ej|                  d-      f   ZMee8 ejx                  d.      f   ZNee8 ejx                  d/      f   ZOeeAe8   dz   ejx                  d0      f   ZPeeAe8    ejx                  d1      f   ZQee8 ejx                  d2      f   ZReeAe8   dz   ejx                  d3      f   ZSee8 ejx                  d4      f   ZT e1d5      ZUd6ed7eddfd8ZVeUj                  d9d:d;ig d<=      	 	 	 	 	 	 	 	 	 	 	 	 	 dd>e=d?eQd@e%dAe'dBeBdCe,dDe$dEe&dFe@dGeCdHeEdIeHdJeIdeFdKe+ddf dL       ZXeUj                  dMg dNO      	 	 	 	 ddeRdPeeD ej|                  dQdRdS      f   dTeeGdz   ej|                  dUdVdW      f   deFdKe+ddfdX       ZYdYeZe8e8f   dZeAe9e8e8e8f      deDfd[Z[d\eGddfd]Z\de8d^eeZe8ef      d_eAe8   dee9eDe8eAeAe8eGz        f      fd`Z]eUj                  dadbgO      	 	 	 ddceSdeFdKe+ddfdd       Z^eUj                  dedfdggO      	 	 	 	 ddheeD ej|                  didjdk      f   deFdKe+dleeAe8   dz   ej|                  dQdmdn      f   ddf
do       Z_eUj                  dpdqgO      ddr       Z`eUj                  dsdtgO      	 	 ddceeAe8    ejx                  du      f   deFdKe+ddfdv       ZaeUj                  dwdxgO      	 	 ddeRdeFdKe+ddfdy       ZbeUj                  dzg d{O      	 	 	 ddceeAe8    ejx                  d|      f   dGee8dz   ej|                  d}      f   deFdKe+ddf
d~       ZceUj                  dddgO      	 	 	 	 ddeRdBeBdeeD ej|                  dd      f   deFdKe+ddfd       ZdeUj                  dJg dO      	 	 	 	 ddeRde*de)deFdKe+ddfd       Ze e1d      ZfeUj                  efd       efj                  d9d:d;ig d=      	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddeOdePd>e?dFe@d@e%dAe'dBeBdCe,dDe$dEe&dGeCdHeEdIeHdJeIdeFdKe+deJdeKddf&d       Zh e1d      ZieUj                  eid       eij                  d9d:d;idg=      	 	 	 	 	 	 	 	 	 	 	 	 	 ddeNd>e=d?eQdeLdeMd@e%dAe'dBeBdCe,dDe$dEe&dFe@dGeCdIeHdeFdKe+ddf"d       Zjeij                  dedgO      	 	 	 	 ddheeD ej|                  didjd      f   deFdKe+dleeAe8   dz   ej|                  dQdmdn      f   ddf
d       Zkeij                  dsdgO      	 	 ddeeAe8    ejx                  d      f   deFdKe+ddfd       Zleij                  ddgO      	 	 ddeTdeFdKe+ddfd       Zmeij                  ddgO      	 	 ddeTdeFdKe+ddfd       Zneij                  ddgO      	 	 ddeTdeFdKe+ddfd       Zoeij                  dddgO      	 	 	 	 ddeTdBeBdeeD ej|                  dd      f   deFdKe+ddfd       Zp e1d      Zqeij                  eqd       eqj                  d9d:d;iddg=      	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddeNdeOdePdeLdeMd>e?dFe@d@e%dAe'dBeBdCe,dDe$dEe&dGeCdIeHdeFdKe+deJdeKddf(d       ZrdeAe8   dz  deZe8e8f   dz  fdZsdeAeAe8eGz        deAe8   de8fdZt ed      Zudeeu   dedeeu   f   deZddfdZvdej                  j                  dedeeu   f   deAeZ   deeu   fdZyy)a  Contains commands to interact with jobs on the Hugging Face Hub.

Usage:
    # run a job
    hf jobs run <image> <command>

    # List running or completed jobs
    hf jobs ps [-a] [-f key=value]

    # Print logs from a job (non-blocking)
    hf jobs logs <job-id>

    # Stream logs from a job (blocking, like `docker logs -f`)
    hf jobs logs -f <job-id>

    # Stream resources usage stats and metrics from a job
    hf jobs stats <job-id>

    # Inspect detailed information about a job
    hf jobs inspect <job-id>

    # Cancel a running job
    hf jobs cancel <job-id>

    # Wait until one or more jobs finish
    hf jobs wait <job-id> [<job-id>...]

    # List available hardware options
    hf jobs hardware

    # Run a UV script
    hf jobs uv run <script>

    # Schedule a job
    hf jobs scheduled run <schedule> <image> <command>

    # List scheduled jobs
    hf jobs scheduled ps [-a] [-f key=value]

    # Inspect a scheduled job
    hf jobs scheduled inspect <scheduled_job_id>

    # Suspend a scheduled job
    hf jobs scheduled suspend <scheduled_job_id>

    # Resume a scheduled job
    hf jobs scheduled resume <scheduled_job_id>

    # Delete a scheduled job
    hf jobs scheduled delete <scheduled_job_id>

    N)CallableIterable)fnmatch)EmptyQueue)	AnnotatedAnyTypeVar)urlsplit)HfApiJobHardwareJobInfoJobStage)TERMINAL_JOB_STAGES)CLIError)logging)_format_size)format_durationparse_duration   )
EnvFileOptEnvOptSecretsFileOpt
SecretsOpt
SoftChoiceSshDryRunOptSshIdentityFileOptTokenOpt
VolumesOptexec_ssh
get_hf_apiparse_env_mapparse_volumestyper_factory)_dataclass_to_dictoutjob_id	namespacereturnc                     | st        d      | j                  d      dkD  rt        d|  d      d| vr| |fS | j                  dd      \  }}|r|st        d|  d      |||k7  rt        d| d| d      ||fS )	a&  Extract namespace from job_id if provided in 'namespace/job_id' format.

    Allows users to pass job IDs copied from the Hub UI (e.g. 'username/job_id')
    instead of only bare job IDs. If the namespace is also provided explicitly via
    --namespace and conflicts, a CLIError is raised.
    zJob ID cannot be empty./r   z<Job ID must be in the form 'job_id' or 'namespace/job_id': 'z'.z(Conflicting namespace: got --namespace='z ' but job ID implies namespace='')r   countsplit)r'   r(   extracted_namespaceparsed_job_ids       Z/var/www/html/tokenscope/api/venv/lib/python3.12/site-packages/huggingface_hub/cli/jobs.py_parse_namespace_from_job_idr2   l   s     011||C1UV\U]]_`aa
&y  )/c1)=&mUV\U]]_`aa.A!A6ykAabuavvwx
 	
 ---    g?zThe Docker image to use.)helpz.Use a custom Docker image with `uv` installed.zcFlavor for the hardware. Run 'hf jobs hardware' to list available flavors. Defaults to `cpu-basic`.)r4   
click_typez-lz--labelz3Set labels. E.g. --label KEY=VALUE or --label LABELzVMax duration: int/float with s (seconds, default), m (minutes), h (hours) or d (days).z-dz--detachz3Run the Job in the background and print the Job ID.zVThe namespace where the job will be running. Defaults to the current user's namespace.z--exposezExpose a container port through the jobs proxy. Repeat the flag for multiple ports (e.g. `--expose 8000 --expose 8001`). Each exposed port is reachable on the public jobs domain; access requires an HF token with read access to the job's namespace.z--sshzMake the job's container reachable over SSH. Connect with `hf jobs ssh <job_id>`. Requires an SSH public key registered on https://huggingface.co/settings/keys.z--withz%Run with the given packages installedz-pz--pythonz5The Python interpreter to use for the run environmentz!Suspend (pause) the scheduled Jobz8Allow multiple instances of this Job to run concurrentlyzWOne of annually, yearly, monthly, weekly, daily, hourly, or a CRON schedule expression.z$UV script to run (local file or URL)zArguments for the scriptzThe command to run.zJob ID (or 'namespace/job_id')zJob IDs (or 'namespace/job_id')z2Scheduled Job ID (or 'namespace/scheduled_job_id')zRun and manage Jobs on the Hub.apijobc                 X   | j                  |j                  |j                  j                  d      D ]  }t	        j
                  |        | j                  |j                  |j                  j                        }|j                  j                  t        j                  k7  rc|j                  j                  rd|j                  j                   nd}t        d|j                   d|j                  j                   d|       t	        j
                  d|j                   d	       y
)zcStream Job logs until the Job ends, then fail the command if the Job did not complete successfully.T)r'   r(   followr'   r(   z:  zJob z finished with stage 'r,   z
 completedN)fetch_job_logsidownernamer&   textwait_for_jobstatusstager   	COMPLETEDmessager   )r6   r7   logfinalrE   s        r1   _stream_logs_and_check_statusrH   &  s    !!399>>RV!W  CFFciinnEE||X///161E1EBu||++,-2ehhZ'=ell>P>P=QQRSZR[\]]HHtEHH:Z()r3   runignore_unknown_optionsT)z3hf jobs run python:3.12 python -c 'print("Hello!")'z1hf jobs run --detach python:3.12 python script.pyz3hf jobs run -e FOO=foo python:3.12 python script.pyz;hf jobs run --secrets HF_TOKEN python:3.12 python script.pyz^hf jobs run -v hf://org/my-model:/data -v hf://buckets/org/b:/mnt python:3.12 python script.py)context_settingsexamplesimagecommandenvsecretslabelvolumeenv_filesecrets_fileflavortimeoutdetachexposesshtokenc                 ,   t        ||      }t        ||      }t        |      }|j                  | |||t        |      t	        |      ||	|||      }t        j                  d|j                  |j                         t        |j                  j                  t              rDdj                  d |j                  j                  D              }t        j                  d|        t        |j                  j                  t               r:t        j                  d|j"                  j$                   d	|j                   d
       |
rB|j"                  j$                   d	|j                   }t        j                  d| d| d       yt'        ||       y)z
Run a Job.rZ   )rM   rN   rO   rP   labelsvolumesrU   rV   rX   rY   r(   Job startedr=   url
c              3   &   K   | ]	  }d |   ywz  N .0ra   s     r1   	<genexpr>zjobs_run.<locals>.<genexpr>b       F2cUF   SExposed ports are reachable at (requires an HF token with read access to the job):
Use `hf jobs ssh r+   &` to open an SSH session into the job.Use `hf jobs logs -f &` to stream logs, or `hf jobs inspect ` to check status.N)r"   r!   run_job_parse_labels_mapr#   r&   resultr=   ra   
isinstancerB   expose_urlslistjoinhintssh_urlstrr>   r?   rH   )rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   r(   rZ   env_mapsecrets_mapr6   r7   urlsjob_refs                        r1   jobs_runr   2  sQ   : C*G6K
5
!C
++ 'f%  C JJ}SWW5#**(($/yyFszz/E/EFFghlgmno#**$$c*$SYY^^$4AcffX=cdeYY^^$AcffX.(	1WX_W``rst!#s+r3   logs)zhf jobs logs <job_id>zhf jobs logs -f <job_id>zhf jobs logs --tail 20 <job_id>z#hf jobs logs -f --tail 100 <job_id>)rL   r9   z-fz--followzqFollow log output (stream until the job completes). Without this flag, only currently available logs are printed.tailz-nz--tailzvNumber of lines to show from the end of the logs. When combined with --follow, starts streaming from the last N lines.c                     t        | |      \  } }t        |      }|j                  | |||      }|D ]  }t        j                  |        |r%|r| d|  n| }t        j
                  d| d       yy)a  Fetch the logs of a Job.

    By default, prints currently available logs and exits (non-blocking).
    Use --follow/-f to stream logs in real-time until the job completes.
    Use --tail/-n to limit the number of lines returned (server-side when supported).

    Note: following exits when the log stream ends, regardless of whether the Job
    succeeded or failed. Run `hf jobs inspect <job_id>` to check the final status.
    r\   )r'   r(   r9   r   r+   z#Stream ended. Run `hf jobs inspect z6` to check the final status (e.g. COMPLETED or ERROR).N)r2   r!   r<   r&   r@   rx   )	r'   r9   r   r(   rZ   r6   r   rF   r~   s	            r1   	jobs_logsr   m  s    N 5VYGFI
5
!CVyVZ[D -6YKq)F6wi?uvw r3   job_propertiesfiltersc                     |D ]Z  \  }}}| j                  |      }||dk(  r  yt        |j                         |j                               }|dk(  r|r	|dk(  sW|sZ y y)z5Check if scheduled job matches all specified filters.!=F=T)getr   lower)r   r   keyop_strpatternvaluematchs          r1   _matches_filtersr     sk     ' VW""3'=~w}}7cM%Vt^ r3   nc                 F    d}d}t        |       D ]  }t        ||        y )Nz[1Az[2K)end)rangeprint)r   LINE_UP
LINE_CLEARis       r1   _clear_liner     s)    GJ1X 'g:&'r3   metrics_streamtable_headersc              #     K   |D ]J  }| |d    dt        |d   dz  d      t        d|d   z  |d   z  d	       dt        |d          d
t        |d          dt        |d          dt        |d          dg}|d   rt        |d   t              r|gdgt	        |      z  gt	        |d         dz
  z  z   }t        |t        |d               D ]R  \  }}|d   |   }||d    dt        d|d   z  |d   z  d	       dt        |d          d
t        |d          dgz  }T n!|dgt	        |      t	        |      z
  z  z  }|g}d| |f M d| g f y w)Ncpu_usage_pct%cpu_millicoresg     @@r   d   memory_used_bytesmemory_total_bytes   zB / Brx_bpszbps / tx_bpsbpsgpusr;   utilizationN/AFT)roundr   rt   dictlenzipsorted)r'   r   r   metricsrowrowsgpu_idgpus           r1   _get_jobs_stats_rowsr     s     " "'(*'*+f4a8S7#677'BV:WWYZ[\\]^G$789:$|GThLi?j>kklmGH-./vl78CT6U5VVYZ
 6?z'&/4@5RD3s8O,GFO0Dq0HIID"4)@A Vfof-=)*!,S3':#;;cBV>WWYZ[\\]^#C(;$<=>d<PSThPiCjBkklm  E7c-03s8;<<C5DVT!!+", 
s   EEstatszhf jobs stats <job_id>job_idsc           	         | +g }| D ]"  }t        ||      \  }}|j                  |       $ |} t        |      }||j                         d   }| O|j	                  |      D cg c]4  }|j
                  r|j
                  j                  nddv r|j                  6 } }t        |       dk(  rt        j                  d       yg d	}t        j                  j                  t        |             5 }i }	| D ]+  }|g}
|
|d
d D cg c]  }d|v sd|v rdnd c}z  }
|
g|	|<   - t        j                         }|	D 
cg c]  }|	|   D ]  }
|
  }}}
t        t!        ||             | D cg c]  }||j#                  ||      |d }}t%        |t&        |      D ]  \  }}}|r|	j)                  |d       n||	|<   t        j                         }||z
  t*        k\  sBt-        dt        |      z          |	D 
cg c]  }|	|   D ]  }
|
  }}}
t        t!        ||             |} 	 ddd       yc c}w c c}w c c}
}w c c}w c c}
}w # 1 sw Y   yxY w)z7Fetch the resource usage statistics and metrics of JobsNr\   r?   r(   UNKNOWNRUNNINGUPDATINGr   zNo running jobs found)	zJOB IDzCPU %zNUM CPUzMEM %z	MEM USAGEzNET I/Oz
GPU UTIL %z	GPU MEM %zGPU MEM USAGEr   r+   USAGEz-- / --z--)headersr:   )r'   r   r   )kwargs_listr   )r2   appendr!   whoami	list_jobsrB   rC   r=   r   r&   r@   multiprocessingpool
ThreadPooltimer   	_tabulatefetch_job_metricsiflatmap_unorderedr   popSTATS_UPDATE_MIN_INTERVALr   )r   r(   rZ   
parsed_idsr'   r6   r7   r   r   rows_per_job_idr   headerlast_update_time
total_rowsr   doner   nows                     r1   
jobs_statsr     s    
 	&F <VY OFIf%	& 
5
!CJJL(	 }}y}9
$'JJ

  IBYY FF
 

 7|q()
M 
			(	(W	6 '$<> 	,F$*8C^klmln^opTZ#-7f3DI4OppC'*eOF#	,  99;(7[f?SYCZ[Cc[c[
[ 	i
M:; "
 	 !"%"7"7vQZ"7"[!.
 
 #5T;O]h"i 
	'D&$##FD1*.'))+C%%)BBAJ/00?cf?[aKbcCcccc
ci
MBC#& 
	')' ''
. q \

  d9' 'sO   %9H60IH;
&I<I I,I
AI I<I
I;IIpsz
hf jobs pszhf jobs ps -aallz-az--allz*Show all Jobs (default shows just running)filterz--filterz>Filter output based on conditions provided (format: key=value)c                    t        |      }|j                  |      }g }g }|xs g D ]`  }|j                  d      s|j                  d      r|j                  d      r3|t        d      d }	d|	v rt	        j
                  d|	 d       c|	d	d
}}}
nG|t        d      d }	d|	v r|	j                  dd      \  }
}n|	d
}}
|
j                  d      rd	}|
dd }
nd}|j                  |
j                         ||j                         f       d|v rb|j                  dd      \  }}|j                  d      rd	}|dd }nd}|j                  |j                         ||j                         f       Ht	        j
                  d| d       c g }|D ]  }|j                  r|j                  j                  nd}| s|dvr.|j                  xs d}|j                  xs g }|rdj                  |      nd}|j                  ||j                         |d}t!        ||      st!        |j"                  xs i |      s|j                  |        g }|D ]  }t%        |      }|j'                  d      xs i }|j'                  d      xs g }|j'                  dd      |d<   |j'                  d      xs d|d<   |rdj                  |      nd|d<   |j'                  d      r|d   dd j)                  dd      nd|d<   |j'                  d       xs i j'                  d!d      |d <   t+        |j'                  d"            |d#<   |j                  |        t	        j,                  |g d$d%       |sO|r1d&j                  d' |D              }t	        j.                  d(|        y| s|st	        j0                  d)       yyyy)*z
List Jobs.r\   r   zlabel!=zlabel=Nr   z-Ignoring invalid label filter format 'label!=z'. Use label!=key format.r   *r   ! Ignoring invalid filter format ''. Use key=value format.r   r   r    )r=   rM   rB   rN   	durationsrN   r=   r;   r'   docker_imageimage/space
created_at   TcreatedrB   rC   running_secsruntime)r'   r   rN   r   rB   r   r   id_key, c              3   4   K   | ]  \  }}}| | |   y wNre   rg   kovs       r1   rh   zjobs_ps.<locals>.<genexpr>v  s"     #HWQ1qc!QCL#H   zNo jobs matched filters: zHNo running jobs. Use `-a`/`--all` to include finished (and failed) jobs.)r!   r   
startswithr   r&   warningr.   endswithr   r   rB   rC   r   rN   rw   r=   r   r]   r%   r   replacer   tabler@   rx   )r   r(   rZ   r   r6   jobsr   labels_filtersf
label_part	label_keyoplabel_valuer   r   filtered_jobsr7   rB   image_or_spacecmdcommand_strpropsitemsitemr   filters_msgs                             r1   jobs_psr    s   , 5
!C==9=-D*,G13N\r X<<	"all8&<||I&s9~/0
*$KK"OPZ|[t uv-7s{2	s8}/
*$-7-=-=c1-E*I{-7{I%%c*B )#2IB!!9??#4b+:K:K:M"NOAXaJC||C #2hNNCIIKU[[];<KK:1#=UVW?XD M "%(ZZ!!Yv%<<))2UkkR'*chhsmvv&,,.]hiw/

 0b.AS!" #%E 
!#&HH[)/R	hhy!'R$+X"hh~6?%]+.#((3-EYGKxxP\G]$|,Sb199#sCchY((8,277KX))--*GHYT
 IIT
 ))#H#HHKHH0>?^HH_` ,	 r3   hardwarezhf jobs hardwarec                  ^   t               } | j                         }g }|D ]  }d}|j                  rG|j                  j                   d|j                  j                   d|j                  j
                   d}|j                  rd|j                  dnd}|j                  rd|j                  dz  d	nd}|j                  |j                  |j                  |j                  |j                  |j                  |||d
        t        j                  |       t        j                  d       y)z(List available hardware options for Jobsr;   zx  ()$z.4ffree<   z.2f)r?   zpretty namecpuramstorageacceleratorzcost/minz	cost/hourzLUse `hf jobs run --flavor <name> ...` to request a specific hardware flavor.N)r!   list_jobs_hardwarer  quantitymodelvramunit_cost_usdr   r?   pretty_namer  r  ephemeral_storager&   r   rx   )r6   hardware_listr  hwaccelerator_infocost_min	cost_hours          r1   jobs_hardwarer  |  s    ,C**,ME 
>>"$.."9"9!:"R^^=Q=Q<RRTUWUcUcUhUhTiijk131A1AQr'',-v797G7Ga((2-c23V	!~~vvvv///$&		

$ IIeHH[\r3   inspectzhf jobs inspect <job_id>z*Job IDs to inspect (or 'namespace/job_id')c                    g }| D ]"  }t        ||      \  }}|j                  |       $ |} t        |      }| D cg c]  }|j                  ||       }}t	        j
                  |D cg c]  }t        |       c}       yc c}w c c}w )z0Display detailed information on one or more Jobsr\   r:   N)r2   r   r!   inspect_jobr&   r   r%   )r   r(   rZ   r   r'   r6   r   r7   s           r1   jobs_inspectr"    s     J "8K	&!" G
5
!CNUVFCOO6YO?VDVII$73!#&78 W7   B+B
cancelzhf jobs cancel <job_id>c                     t        | |      \  } }t        |      }|j                  | |       t        j                  d|        y)zCancel a Jobr\   r:   zJob cancelledr=   N)r2   r!   
cancel_jobr&   rs   )r'   r(   rZ   r6   s       r1   jobs_cancelr(    s=     5VYGFI
5
!CNN&IN6JJ6*r3   wait)zhf jobs wait <job_id>z"hf jobs wait <job_id_1> <job_id_2>z"hf jobs ps -q | xargs hf jobs waitz,Job IDs to wait for (or 'namespace/job_id').zZMax time to wait: int/float with s (seconds, default), m (minutes), h (hours) or d (days).c           
      *   g }t               }| D ]3  }t        ||      \  }}|j                  |       |j                  |       5 t	        |      dkD  r3t        ddj                  d t        |t              D              z         |j                         }|t        |      nd}	t        |      }
t        j                  dt	        |       d	      }	 |
j                  ||	|
      }|j!                  t	        |       d       t        j"                  |D cg c]C  }|j$                  t        |j                  j&                        |j                  j(                  dE c}       |D cg c],  }|j                  j&                  t*        j,                  k7  s+|. }}|r<t        t	        |       dt	        |       ddj                  d |D              z         y# t        $ r" |j!                  d       t        d| d      dw xY wc c}w c c}w )a  Wait for one or more Jobs to reach a terminal state.

    Blocks until every Job has finished, then exits with code 0 if all Jobs completed
    successfully, or a non-zero exit code if any Job was canceled, errored or deleted.

    All Jobs must belong to the same namespace.
    r   z0All Job IDs must be in the same namespace, got: r   c              3   2   K   | ]  }t        |        y wr   )rz   )rg   nss     r1   rh   zjobs_wait.<locals>.<genexpr>  s     FBBFs   )r   Nr\   zWaiting for z Job(s) to finish...)rV   r(   z
Timed out.zTimed out after z waiting for Job(s) to finish.z Job(s) finished.)r=   rC   rE   z of z' Job(s) did not complete successfully: c              3   h   K   | ]*  }|j                    d |j                  j                   d , yw)r	  r
  N)r=   rB   rC   )rg   r7   s     r1   rh   zjobs_wait.<locals>.<genexpr>  s,     KS366("SZZ%5%5$6a8Ks   02)setr2   r   addr   r   rw   r   rz   r   r   r!   r&   rB   rA   TimeoutErrorr   r   r=   rC   rE   r   rD   )r   rV   r(   rZ   r   
namespacesr'   	parsed_idparsed_namespacetimeout_secsr6   rB   r   r7   faileds                  r1   	jobs_waitr6    s   > JJ )&B69&U#	#)$'() :>iiF&*EFFG
 	
  I.5.A>'*tL
5
!CZZ,s:&77KLMF]
LIV KK3t9+./0IIhlmadcffs3::+;+;'<I[I[\mn!LcSZZ%5%59K9K%KcLFL6{m4D	{*QRiiKFKKL
 	
   ]L!)'2PQRX\\]
 nLs   G AH+,HH+Hr]   z8hf jobs labels <job_id> --label env=prod --label team=mlzhf jobs labels <job_id> --clearclearz--clearzRemove all labels from the job.c                     |s|st        d      |r|rt        d      t        | |      \  } }t        |      xs i }t        |      }|j	                  | ||      }t        j                  d|j                         y)z5Update labels on a Job. Replaces all existing labels.OPlease set at least one label with --label. To remove all labels, pass --clear.bCannot set labels and clear them at the same time. Please use either --label or --clear, not both.r\   )r'   r]   r(   Labels updatedr&  N)r   r2   rr   r!   update_job_labelsr&   rs   r=   )r'   rQ   r7  r(   rZ   r]   r6   r7   s           r1   jobs_labelsr=    s     hiip
 	
 5VYGFIu%+F
5
!C


vf	

RCJJCFF+r3   )zhf jobs ssh <job_id>zhf jobs ssh <job_id> --dry-runz)hf jobs ssh <job_id> -i ~/.ssh/id_ed25519identity_filedry_runc                    t        | |      \  } }t        |      }|j                  | |      }|j                  j                  t        d      |j                  j                  t        v r0t        d|j                   d|j                  j                   d      |j                  j                  t        j                  k7  rt        j                  d|j                   d	|j                  j                   d
      }|j                  |j                  |t        j                  g      }|j                  j                  t        j                  k7  rA|j                  d       t        d|j                   d|j                  j                   d      |j                  d       t        |j                  j                        }t        |j                    d|j"                   |j$                  ||       y)a   SSH into a running Job.

    If the Job is not yet running, waits until it reaches the RUNNING state before
    connecting. Requires the Job to be started with SSH enabled (`hf jobs run --ssh ...`)
    and your SSH public key to be registered at https://huggingface.co/settings/keys.
    r\   r:   Nz[SSH is not enabled on this job. Start a job with SSH support using `hf jobs run --ssh ...`.zCannot SSH into job 'z%': job has already finished (stage: 'z').zWaiting for job 'z' to be running (stage: 'z')...)r'   r(   stageszJob finished.z1': job finished before reaching RUNNING (stage: 'zJob is running.@)portr>  r?  )r2   r!   r!  rB   ry   r   rC   r   r=   r   r   r&   rA   r   r   r    usernamehostnamerC  )	r'   r>  r?  r(   rZ   r6   r7   rB   ry   s	            r1   jobs_sshrF    s   * 5VYGFI
5
!C
//9/
=C
zz!tuu
zz...svvh6[\_\f\f\l\l[mmpqrr
zz8+++/x7PQTQ[Q[QaQaPbbghicff	8K[K[J\]::x///KK('x/`adakakaqaq`rruv  	%&szz))*G
Ag../0\\#	r3   zFRun UV scripts (Python with inline dependencies) on HF infrastructure.uv)r?   )zhf jobs uv run my_script.pyz$hf jobs uv run --detach my_script.pyz1hf jobs uv run ml_training.py --flavor a10g-smallz+hf jobs uv run --with transformers train.pyzNhf jobs uv run -v hf://org/my-model:/data -v hf://buckets/org/b:/mnt script.pyscriptscript_argswith_pythonc                 :   t        ||      }t        ||	      }t        |      }|j                  | |xs g |||||t        |      t	        |      ||
|||      }t        j                  d|j                  |j                         t        |j                  j                  t              rDdj                  d |j                  j                  D              }t        j                  d|        t        |j                  j                  t               r:t        j                  d|j"                  j$                   d	|j                   d
       |rB|j"                  j$                   d	|j                   }t        j                  d| d| d       yt'        ||       y)8Run a UV script (local file or URL) on HF infrastructurer\   )rH  rI  dependenciesrK  rM   rO   rP   r]   r^   rU   rV   rX   rY   r(   r_   r`   rb   c              3   &   K   | ]	  }d |   ywrd   re   rf   s     r1   rh   zjobs_uv_run.<locals>.<genexpr>~  ri   rj   rk   rl   r+   rm   rn   ro   rp   N)r"   r!   
run_uv_jobrr   r#   r&   rs   r=   ra   rt   rB   ru   rv   rw   rx   ry   rz   r>   r?   rH   )rH  rI  rM   rU   rO   rP   rQ   rR   rS   rT   rV   rW   rX   rY   r(   rZ   rJ  rK  r{   r|   r6   r7   r}   r~   s                           r1   jobs_uv_runrQ  H  s`   @ C*G6K
5
!C
..%2 'f%  C  JJ}SWW5#**(($/yyFszz/E/EFFghlgmno#**$$c*$SYY^^$4AcffX=cdeYY^^$AcffX.(	1WX_W``rst!#s+r3   z,Create and manage scheduled Jobs on the Hub.	scheduledz>hf jobs scheduled run "0 0 * * *" python:3.12 python script.pyschedulesuspendconcurrencyc                 4   t        ||	      }t        ||
      }t        |      }|j                  ||| ||||t        |      t	        |      ||||      }t        j                  d|j                         t        j                  d|j                   d       y)zSchedule a Job.r\   )rM   rN   rS  rT  rU  rO   rP   r]   r^   rU   rV   rX   r(   Scheduled Job createdr&  Use `hf jobs scheduled inspect ` to view its details.N)	r"   r!   create_scheduled_jobrr   r#   r&   rs   r=   rx   )rS  rM   rN   rT  rU  rO   rP   rQ   rR   rS   rT   rU   rV   rX   r(   rZ   r{   r|   r6   scheduled_jobs                       r1   scheduled_runr\    s    0 C*G6K
5
!C,, 'f% - M JJ&=+;+;<HH.}/?/?.@@VWXr3   zhf jobs scheduled psz1Show all scheduled Jobs (default hides suspended)c                 n   t        |      }|j                  |      }g }|xs g D ]  }d|v ra|j                  dd      \  }}	|j                  d      rd}
|dd }nd}
|j	                  |j                         |
|	j                         f       ht        j                  d	| d
        g }|D ]  }|j                  xs d}| s|r|j                  j                  xs d}|j                  j                  xs g }|rdj                  |      nd}|j                  |t        |      |d}t        ||      s|j	                  |        g }|D ]  }t!        |      }|j#                  d      xs i }|j#                  d      xs i }|j#                  d      }|j#                  d      xs g }|j#                  d      xs d|d<   |rdj                  |      nd|d<   |r)|j#                  d      r|d   dd j%                  dd      nd|d<   |j#                  d      r|d   dd j%                  dd      nd|d<   |j#                  d      xs d|d<   |j	                  |        t        j&                  |g dd       |s4|r1dj                  d  |D              }t        j(                  d!|        yyy)"zList scheduled Jobsr\   r   r   r   r   r   Nr   r   r   Fr   r   )r=   rM   rT  rN   job_specrB   last_jobrN   r   r   atr   r   last_runnext_job_run_atnext_runrT  )r=   rS  r   rN   ra  rc  rT  r=   r   r   c              3   4   K   | ]  \  }}}| | |   y wr   re   r   s       r1   rh   zscheduled_ps.<locals>.<genexpr>  s"     DAq1#aSDr   z#No scheduled jobs matched filters: )r!   list_scheduled_jobsr.   r   r   r   r&   r   rT  r^  r   rN   rw   r=   rz   r   r%   r   r   r   r@   )r   r(   rZ   r   r6   scheduled_jobsr   r   r   r   r   r   r[  rT  r   r   r  r  r  sjr  r^  status_dictr_  r  s                            r1   scheduled_psri    s   , 5
!C,,y,AN*,G\r X!8aJC||C #2hNNCIIKU[[];<KK:1#=UVWX M' 
,''05w&//<<E$$,,2'*chhsm$''.SQX\fqrw/]+
, #%E !"%88J'-2hhx(.B??:.ll9%+&ll>:Ce]+.#((3-EYDLQYQ]Q]^bQc8D>#2.66sC@inZEP__UfEgK)*3B/77SAmr 	Z ((9-6YT II_
 WiiDGDD6{mDE 5r3   zhf jobs scheduled inspect <id>scheduled_job_idsz>Scheduled Job IDs to inspect (or 'namespace/scheduled_job_id')c                    g }| D ]"  }t        ||      \  }}|j                  |       $ |} t        |      }| D cg c]  }|j                  ||       }}t	        j
                  |D cg c]  }t        |       c}       yc c}w c c}w )z:Display detailed information on one or more scheduled Jobsr\   scheduled_job_idr(   N)r2   r   r!   inspect_scheduled_jobr&   r   r%   )	rj  r(   rZ   r   r'   r6   rm  rf  r[  s	            r1   scheduled_inspectro    s     J# "8K	&!" #
5
!C !2 	!!3Cy!YN  IInU]!-0UV	 Vr#  deletezhf jobs scheduled delete <id>rm  c                     t        | |      \  } }t        |      }|j                  | |       t        j                  d|        y)zDelete a scheduled Job.r\   rl  zScheduled Job deletedr&  N)r2   r!   delete_scheduled_jobr&   rs   rm  r(   rZ   r6   s       r1   scheduled_deletert  $  F     #??OQZ"[i
5
!C.>)TJJ&+;<r3   zhf jobs scheduled suspend <id>c                     t        | |      \  } }t        |      }|j                  | |       t        j                  d|        t        j
                  d|  d       y)z Suspend (pause) a scheduled Job.r\   rl  zScheduled Job suspendedr&  zUse `hf jobs scheduled resume z` to resume it.N)r2   r!   suspend_scheduled_jobr&   rs   rx   rs  s       r1   scheduled_suspendrx  1  s^     #??OQZ"[i
5
!C/?9UJJ(-=>HH-.>-?OPr3   resumezhf jobs scheduled resume <id>c                     t        | |      \  } }t        |      }|j                  | |       t        j                  d|        y)z!Resume (unpause) a scheduled Job.r\   rl  zScheduled Job resumedr&  N)r2   r!   resume_scheduled_jobr&   rs   rs  s       r1   scheduled_resumer|  ?  ru  r3   z>hf jobs scheduled labels <id> --label env=prod --label team=mlz%hf jobs scheduled labels <id> --clearz)Remove all labels from the scheduled job.c                     |s|st        d      |r|rt        d      t        | |      \  } }t        |      xs i }t        |      }|j	                  | ||      }t        j                  d|j                         y)z?Update labels on a scheduled Job. Replaces all existing labels.r9  r:  r\   )rm  r]   r(   r;  r&  N)r   r2   rr   r!   update_scheduled_job_labelsr&   rs   r=   )rm  rQ   r7  r(   rZ   r]   r6   r[  s           r1   scheduled_labelsr  L  s     hiip
 	
 #??OQZ"[iu%+F
5
!C33)&I 4 M JJM$4$45r3   z)Schedule UV scripts on HF infrastructure.z.hf jobs scheduled uv run "0 0 * * *" script.pyz<hf jobs scheduled uv run "0 0 * * *" script.py --with pandasc                 B   t        ||      }t        ||      }t        |      }|j                  ||xs g | |||||||t        |	      t	        |
      ||||      }t        j                  d|j                         t        j                  d|j                   d       y)rM  r\   )rH  rI  rS  rT  rU  rN  rK  rM   rO   rP   r]   r^   rU   rV   rX   r(   rW  r&  rX  rY  N)	r"   r!   create_scheduled_uv_jobrr   r#   r&   rs   r=   rx   )rS  rH  rI  rT  rU  rM   rU   rO   rP   rQ   rR   rS   rT   rV   rX   r(   rZ   rJ  rK  r{   r|   r6   r7   s                          r1   scheduled_uv_runr  n  s    < C*G6K
5
!C

%
%%2 'f%! & C$ JJ&3662HH.svvh6LMNr3   c                 b    | syi }| D ]$  }d|v r|j                  dd      n|df\  }}|||<   & |S )a  Parse label key-value pairs from CLI arguments.

    Args:
        labels: List of label strings in KEY=VALUE format. If KEY only, then VALUE is set to empty string.

    Returns:
        Dictionary mapping label keys to values, or None if no labels provided.
    Nr   r   r;   )r.   )r]   
labels_map	label_varr   r   s        r1   rr   rr     sQ     !#J  	03y0@Y__S!,yRTo
U
3  r3   r   r   c           
         t        g | | D cg c]  }t        d |D               }}t        t        j                         j                  t        |      dz        }t        |      t        |      z   |kD  r|j                  t        |            }||xx   dz  cc<   t        |      t        |      z   |k  r#|t        |      z
  t        |      z
  ||   z   ||<   t        |      t        |      z   |kD  r dt        |      z  j                  | }g }|j                   |j                  |        |j                   |j                  |D cg c]  }d|z  	 c}        | D ]r  }	t        |	|      D 
cg c];  \  }
}t        t        |
            |kD  rt        |
      d|dz
   dz   n
t        |
      = }}
}|j                   |j                  |        t d	j                  |      S c c}w c c}w c c}}
w )
z
    Inspired by:

    - stackoverflow.com/a/8356620/593036
    - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data
    c              3   D   K   | ]  }t        t        |              y wr   )r   rz   )rg   xs     r1   rh   z_tabulate.<locals>.<genexpr>  s     /ac#a&k/s       r   z{{:{}}} -N   z...rb   )r   maxshutilget_terminal_sizecolumnsr   sumindexformatr   rz   rw   )r   r   col
col_widthsterminal_widthcol_to_minimize
row_formatlineswr   r  	col_widthrow_format_argss                r1   r   r     s    <?;N;Ng;NOC#/3//OJO113;;S\B=NON
g,Z
(>
9$**3z?;?#)#w<#j/)^;*83z?*JSQX\*Y\fgv\w*wJ'	 g,Z
(>
9
 4*s7|+33Z@JE	LL"""G,-	LL"""j$AS1W$ABC : !$C 4
9 033q6{Y/FCF?Y]#e+CPQFR
 
 	&Z&&89: 99U# P %B
s   G*G/6A G4r   queuefunc.kwargsc                 @     |di |D ]  }| j                  |        y )Nre   )put)r  r  r  rs   s       r1   _write_generator_to_queuer    s#    .. 		&r3   r   r   c          	   #   6  K   t               }|D cg c]  }| j                  t        |||f       }}	 	 	 |j                  d       c c}w # t        $ r' t        d |D              r|j                         rY nY 5w xY wn# t        $ r Y nw xY w	 |D cg c]  }|j                  d       nc c}w c} y# t        j                  $ r Y yw xY w# 	 |D cg c]  }|j                  d       nc c}w c} w # t        j                  $ r Y w w xY wxY ww)a;  
    Takes a function that returns an iterable of items, and run it in parallel using threads to return the flattened iterable of items as they arrive.

    This is inspired by those three `map()` variants, and is the mix of all three:

    * `imap()`: like `map()` but returns an iterable instead of a list of results
    * `imap_unordered()`: like `imap()` but the output is sorted by time of arrival
    * `flatmap()`: like `map()` but given a function which returns a list, `flatmap()` returns the flattened list that is the concatenation of all the output lists
    g?)rV   c              3   <   K   | ]  }|j                           y wr   )ready)rg   async_results     r1   rh   z%iflatmap_unordered.<locals>.<genexpr>  s     N|))+Ns   N)
r   apply_asyncr  r   r   r   emptyKeyboardInterruptr   r0  )r   r  r   r  r  async_resultsr  s          r1   r   r     s     gEfqr\bT%%&?%vAVWrMriii--  s
  NNNSXS^S^S` 	@MN\d+NN++ 			@MN\d+NN++ 		s   D ADB A 
B D+B ;B ?B  B C 	BC BC B; B21	B; :D;CDCDDC=C43	C=<D=DDDDD)NNNNNNNNFNFNN)FNNN)NNN)r)   N)NN)NFNN)NNNNNNNNNNFNFNNNN)NNNNNNNNNNNNN)NNNNNNNNNNNNNNNNN)z__doc__r   multiprocessing.poolr  r   collections.abcr   r   r   r  r   r   typingr   r	   r
   urllib.parser   typerhuggingface_hubr   r   r   r   huggingface_hub._jobs_apir   huggingface_hub.errorsr   huggingface_hub.utilsr   $huggingface_hub.utils._cache_managerr   huggingface_hub.utils._parsingr   r   
_cli_utilsr   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   _outputr%   r&   
get_logger__name__loggerrz   tupler2   r   ArgumentImageArgOptionImageOpt	FlavorOptrv   	LabelsOpt
TimeoutOptbool	DetachOptNamespaceOptint	ExposeOptSshEnabledOptWithOpt	PythonOpt
SuspendOptConcurrencyOptScheduleArg	ScriptArgScriptArgsArg
CommandArgJobIdArg	JobIdsArgScheduledJobIdArgjobs_clirH   rN   r   r   r   r   r   r   r   r  r  r"  r(  r6  r=  rF  uv_app	add_typerrQ  scheduled_appr\  ri  ro  rt  rx  r|  r  scheduled_uv_appr  rr   r   r   r  r   r   r   re   r3   r1   <module>r     s  3j     .   * * !  A A 9 + ) = J     - 
		H	%. .t .cSVY]S]oH^ .8    ENN' $JELL= $JELLrk*	 IELLB	 $JELLe
 ELLB	 $JELLe IELL G	 ELL p IELL4 $JELLD	 4KELL0
 4KELLG ENNf ENN3	 IENN' IENN"
 ENN- IENN.	 ENNA  ?@	*u 	*7 	*t 	* 
	.5  
 #'"-,-,-, 
-, 	-,
 -, -, -, !-, -, -, -, -, 
-, -, -,  
!-,
-,` 

  $ 	 	"'&x&x E	
	&x d
 J	
	&x$ %&x& '&x( 
)&x&xRT#s(^ d5cSVCW>X ]a '3 '4 '!)$sCx.!9JNs)eD#tDsO44568 
'%=$>?"A'A'A' A' 
	A' @A'H 
$,!@A 	" 	%ca	=	
	
ca ca ca S	DQ	
	ca& 
'ca BcaL 
*(:';<] =]6 
)'A&BC #9S	=	
	9 9 9 
9 D9* 
(&?%@A #	+	+	+ 	+ 
		+ B	+ 

  ( 	"4
S	?	
	4
 d
m	
	4
 4
 4
 
4
4
n 
B)   ^c",,, T<5<<	8YZZ[, 	,
 , 
,,* 
	   )-!"##%# # 	#
 # 
##L 
d	e   6  % 	.5  
 "&#'"%3,3,3, 3, 	3,
 
3, 3, 3, 3, 3, !3, 3, 3, 3, 
3, 3,  !3," #3,$ %3,& 
'3,
3,l #QR   ={  3 	.5NO   "&#'"!'Y'Y'Y 'Y 	'Y
  'Y 
'Y 'Y 'Y 'Y 'Y !'Y 'Y 'Y 'Y 'Y  !'Y" 
#'Y
'YT t'=&>? 	" 	%KF	D	
	
KF KF KF S	DQ	
	KF& 
'KF @KF\ y,L+MN #W S	Q	
	W W W 
W OW0 x+J*KL #	='	=	= 	= 
		= M	= y,L+MN #
Q'
Q
Q 
Q 
	
Q O
Q x+J*KL #	='	=	= 	= 
		= M	= H/   hm"6'66 T<5<<	8cdde6 	6
 6 
66. !&QR    (t  4 	.58F   "&"&#'"'-O-O-O -O 	-O
  -O -O -O 
-O -O -O -O -O !-O -O -O  !-O" #-O$ %-O& '-O( 
)-O-Ofd3i$. 4S>D3H $DcCi) DI # 6 CLU1X Xc8A;>N5O Y] bf 




)
)
3#
$ d	
 a[r3   