U
    d                    @   s8  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZ d dlmZ ddlmZmZ ddlmZ ddlmZmZmZmZmZ dd	lmZmZmZ dd
lmZ ddl m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z) ddl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0 ddlm1Z1m2Z2m3Z3 dZ4ej5d  dkZ6e7 dkrbddl8m9Z: eZ;nd dlm<Z: ej;Z;z
e	j=Z=W n e>k
r   dZ=Y nX ej5dkre	j?Z@ne	j@Z@d ZAdZBdZCd ZDdZEdZFdZGdZHd ZIdZJdZKeLeddZMdZNeLedd ZIdZOdZPeQ ZRe	jSZSdd ZTd d! ZUd"d# ZVd$d% ZWdHd&d'ZXG d(d) d)e@ZYG d*d+ d+eZZ[G d,d- d-eZZ\d.d/ Z]G d0d1 d1e^Z_G d2d3 d3e!Z`G d4d5 d5e`ZaG d6d7 d7e`ZbG d8d9 d9e`ZcG d:d; d;e`ZdG d<d= d=e^ZeG d>d? d?e^ZfG d@dA dAefZgG dBdC dCe^ZhG dDdE dEehZiG dFdG dGeeZjdS )I    )absolute_importN)deque)partial   )	cpu_countget_context)util)TERM_SIGNALhuman_statuspickle_loadsreset_signalsrestart_state)	get_errnomem_rsssend_offset)ExceptionInfo)DummyProcess)CoroStopRestartFreqExceededSoftTimeLimitExceeded
TerminatedTimeLimitExceededTimeoutErrorWorkerLostError)EmptyQueuerangevaluesreraise	monotonic)FinalizedebugwarningzEchild process exiting after exceeding memory limit ({0}KiB / {1}KiB)
   Windows)kill_processtree)killg    _B)r#   r#            SIGUSR1g      $@EX_OKi,  皙?c                 C   s>   z
| j }W n tk
r"   d }Y nX |d kr:tt |  S |S N)r   AttributeErrorr   fileno)
connectionZnative r1   1/tmp/pip-unpacked-wheel-lulkwrw3/billiard/pool.py_get_send_offsety   s    

r3   c                 C   s   t t|  S r-   )listmapargsr1   r1   r2   mapstar   s    r8   c                 C   s   t t| d | d S )Nr   r   )r4   	itertoolsstarmapr6   r1   r1   r2   starmapstar   s    r;   c                 O   s   t  j| f|| d S r-   )r   Z
get_loggererror)msgr7   kwargsr1   r1   r2   r<      s    r<   c                 C   s   | t  k	r| | d S r-   )	threadingcurrent_threadstop)threadtimeoutr1   r1   r2   stop_if_not_current   s    rD   c                   @   sb   e Zd ZdZdd Zer<dddZdd	 Zd
d Zdd Z	n"dddZdd	 Zdd Zdd Z	dS )LaxBoundedSemaphorez^Semaphore that checks that # release is <= # acquires,
    but ignores if # releases >= value.c                 C   s   |  j d8  _ |   d S Nr   )_initial_valueacquireselfr1   r1   r2   shrink   s    zLaxBoundedSemaphore.shrinkr   Nc                 C   s   t | | || _d S r-   
_Semaphore__init__rG   rJ   valueverboser1   r1   r2   rN      s    zLaxBoundedSemaphore.__init__c              	   C   s<   | j , |  jd7  _|  jd7  _| j   W 5 Q R X d S rF   )_condrG   _valuenotifyrI   r1   r1   r2   grow   s    zLaxBoundedSemaphore.growc              	   C   s<   | j }|( | j| jk r.|  jd7  _|  W 5 Q R X d S rF   )rR   rS   rG   
notify_allrJ   Zcondr1   r1   r2   release   s
    zLaxBoundedSemaphore.releasec                 C   s   | j | jk rt|  q d S r-   )rS   rG   rM   rX   rI   r1   r1   r2   clear   s    zLaxBoundedSemaphore.clearc                 C   s   t | || || _d S r-   rL   rO   r1   r1   r2   rN      s    c              	   C   s>   | j }|* |  jd7  _|  jd7  _|  W 5 Q R X d S rF   )_Semaphore__condrG   _Semaphore__valuerT   rW   r1   r1   r2   rU      s
    c              	   C   s<   | j }|( | j| jk r.|  jd7  _|  W 5 Q R X d S rF   )rZ   r[   rG   	notifyAllrW   r1   r1   r2   rX      s
    c                 C   s   | j | jk rt|  q d S r-   )r[   rG   rM   rX   rI   r1   r1   r2   rY      s    )r   N)r   N)
__name__
__module____qualname____doc__rK   PY3rN   rU   rX   rY   r1   r1   r1   r2   rE      s   


rE   c                       s0   e Zd ZdZ fddZdd Zdd Z  ZS )MaybeEncodingErrorzVWraps possible unpickleable errors, so they can be
    safely sent through the socket.c                    s.   t || _t || _tt| | j| j d S r-   )reprexcrP   superrb   rN   )rJ   rd   rP   	__class__r1   r2   rN      s    

zMaybeEncodingError.__init__c                 C   s   d| j jt| f S )Nz<%s: %s>)rg   r]   strrI   r1   r1   r2   __repr__   s    zMaybeEncodingError.__repr__c                 C   s   d| j | jf S )Nz)Error sending result: '%r'. Reason: '%r'.)rP   rd   rI   r1   r1   r2   __str__   s     zMaybeEncodingError.__str__)r]   r^   r_   r`   rN   ri   rj   __classcell__r1   r1   rf   r2   rb      s   rb   c                   @   s   e Zd ZdZdS )WorkersJoinedzAll workers have terminated.N)r]   r^   r_   r`   r1   r1   r1   r2   rl      s   rl   c                 C   s
   t  d S r-   )r   )signumframer1   r1   r2   soft_timeout_sighandler   s    ro   c                
   @   s   e Zd ZdddZdd Zdd	 Zd
d ZdddZdd Zdd Z	e
edfddZdd Zdd Zdd ZefddZdd ZdS ) WorkerNr1   Tc                 C   sz   |d ks t |tkr|dks t|| _|| _|| _|| _|| _|| _|	| _	|||  | _
| _| _|
| _|| _| |  d S Nr   )typeintAssertionErrorinitializerinitargsmaxtasksmax_memory_per_child	_shutdownon_exitsigprotectioninqoutqsynqwrap_exceptionon_ready_countercontribute_to_object)rJ   r|   r}   r~   ru   rv   rw   sentinelrz   r{   r   rx   r   r1   r1   r2   rN      s     zWorker.__init__c                 C   s   | j | j| j  |_ |_|_| j j |_| jj |_| jrj| jj |_| jj |_	t
| jj|_nd  |_ |_	|_| j jj|_| jjj|_t
| j j|_|S r-   )r|   r}   r~   _writerr/   inqW_fd_readerZoutqR_fdZsynqR_fdsynqW_fdr3   Zsend_syn_offsetZ_send_syn_offsetsend
_quick_putrecv
_quick_getZsend_job_offset)rJ   objr1   r1   r2   r      s    zWorker.contribute_to_objectc                 C   s6   | j | j| j| j| j| j| j| j| j| j	| j
| jffS r-   )rg   r|   r}   r~   ru   rv   rw   ry   rz   r{   r   rx   rI   r1   r1   r2   
__reduce__  s            zWorker.__reduce__c                    s   t j d gd fdd	}|t _t }|   |   | j|d z`zt | j|d W nD t	k
r } z&t
d| |dd | |d | W 5 d }~X Y nX W 5 | |d d  X d S )	Nc                    s   | d<  | S rq   r1   )status_exitZ	_exitcoder1   r2   exit  s    zWorker.__call__.<locals>.exitpidr   zPool process %r error: %rr   exc_info)N)sysr   osgetpid_make_child_methods
after_forkon_loop_start_do_exitworkloop	Exceptionr<   )rJ   r   r   rd   r1   r   r2   __call__  s    (zWorker.__call__c              	   C   st   |d kr|rt nt}| jd k	r*| || tjdkrfz"| jt	||ff t
d W 5 t| X n
t| d S )Nwin32r   )
EX_FAILUREr+   rz   r   platformr   r   r}   putDEATHtimesleep)rJ   r   exitcoderd   r1   r1   r2   r   +  s    

zWorker._do_exitc                 C   s   d S r-   r1   rJ   r   r1   r1   r2   r   ;  s    zWorker.on_loop_startc                 C   s   |S r-   r1   )rJ   resultr1   r1   r2   prepare_result>  s    zWorker.prepare_resultc              
      s  |p
t  }jj}j}j}j}jp.d}j}	j	}
j
  fdd}d}z|d ksp|r||k r|
 }|rX|\}}|tkst|\}}}}}|t||| ||ff  r||}|sqXzd|	|||f}W n tk
r   dt f}Y nX z|t||||ff W np tk
r } zPt \}}}z8t||d }tt||f}|t||d|f|ff W 5 ~X W 5 d }~X Y nX |d7 }|dkrXt }|dkrtd |dkrX||krXtt|| tW 2S qX|d	| |r||krtntW 
S tW S j|d X d S )
Nr   c                    s`   d}|dkr$t d| jj dd   }|rR|\}}|tkrBdS |tksNtdS |d7 }qd S )Nr   <   z(!!!WAIT FOR ACK TIMEOUT: job:%r fd:%r!!!r   r   FT)r<   r~   r   r/   NACKACKrt   )Zjidireqtype_r7   Z_wait_for_synrJ   r1   r2   wait_for_synM  s     
 z%Worker.workloop.<locals>.wait_for_syn)	completedTFr   z'worker unable to determine memory usagezworker exiting after %d tasks)r   r   r}   r   r   r   rw   rx   r   wait_for_jobr   _ensure_messages_consumedTASKrt   r   r   r   READYr   r   rb   r   r<   r"   MAXMEM_USED_FMTformat
EX_RECYCLEr   r+   )rJ   r!   nowr   r   r   r   rw   rx   r   r   r   r   r   r   Zargs_jobr   funr7   r>   Zconfirmr   rd   _tbwrappedeinfoZused_kbr1   r   r2   r   A  sp    
  
 

zWorker.workloopc                 C   sJ   | j s
dS ttD ]*}| j j|kr2td|  dS tt qtd dS )zr Returns true if all messages sent out have been received and
        consumed within a reasonable amount of time Fz*ensured messages consumed after %d retriesTz<could not ensure all messages were consumed prior to exiting)	r   r   )GUARANTEE_MESSAGE_CONSUMPTION_RETRY_LIMITrP   r!   r   r   ,GUARANTEE_MESSAGE_CONSUMPTION_RETRY_INTERVALr"   )rJ   r   retryr1   r1   r2   r     s    
z Worker._ensure_messages_consumedc                 C   s   t | jdr| jj  t | jdr0| jj  | jd k	rF| j| j  t| j	d t
d k	rftt
t zttjtj W n tk
r   Y nX d S )Nr   r   )full)hasattrr|   r   closer}   r   ru   rv   r   r{   SIG_SOFT_TIMEOUTsignalro   SIGINTSIG_IGNr.   rI   r1   r1   r2   r     s    
zWorker.after_forkc                    s`   |j t|drP|jj t|dr@|jr@|jtffdd	}q\ fdd}nfdd}|S )Nr   get_payloadc                    s   d|  fS NTr1   )rC   loads)r   r1   r2   _recv  s    z'Worker._make_recv_method.<locals>._recvc                    s    | rd fS dS NTFNr1   rC   )_pollgetr1   r2   r     s    
c                    s.   zd | dfW S  t jk
r(   Y dS X d S NTr   r   )r   r   r   )r   r1   r2   r     s    )r   r   r   pollr   r   )rJ   connr   r1   )r   r   r   r2   _make_recv_method  s    
zWorker._make_recv_methodc                 C   s*   |  | j| _| jr |  | jnd | _d S r-   )_make_protected_receiver|   r   r~   r   )rJ   r   r1   r1   r2   r     s
    zWorker._make_child_methodsc                    s2   |  | | jr| jjnd tf fdd	}|S )Nc              
      s   r r| d t tz d\}}|s2W d S W nV ttfk
r } z4t|tjkrbW Y  d S | dt|j t t	W 5 d }~X Y nX |d kr| d t t	|S )Nzworker got sentinel -- exiting      ?zworker got %s -- exiting)

SystemExitr+   EOFErrorIOErrorr   errnoZEINTRrr   r]   r   )r!   readyr   rd   Z_receiveZshould_shutdownr1   r2   receive  s     


z/Worker._make_protected_receive.<locals>.receive)r   ry   is_setr!   )rJ   r   r   r1   r   r2   r     s    
zWorker._make_protected_receive)
NNr1   NNNTTNN)N)r]   r^   r_   rN   r   r   r   r   r   r   r!   r   r   r   r   r   r   r   r   r1   r1   r1   r2   rp      s*               

Mrp   c                       sN   e Zd Zdd Zdd Z fddZdd Zdd
dZdd Zdd Z	  Z
S )
PoolThreadc                 O   s    t |  t| _d| _d| _d S NFT)r   rN   RUN_state_was_starteddaemonrJ   r7   r>   r1   r1   r2   rN     s    
zPoolThread.__init__c              
   C   s   z
|   W S  tk
rX } z0tdt| j|dd tt t t	
  W 5 d }~X Y nB tk
r } z$tdt| j|dd td W 5 d }~X Y nX d S )NzThread %r crashed: %rr   r   )bodyr   r<   rr   r]   _killr   r   r	   r   r   r   r   rJ   rd   r1   r1   r2   run  s    
zPoolThread.runc                    s   d| _ tt| j|| d S r   )r   re   r   startr   rf   r1   r2   r     s    zPoolThread.startc                 C   s   d S r-   r1   rI   r1   r1   r2   on_stop_not_started  s    zPoolThread.on_stop_not_startedNc                 C   s    | j r| | d S |   d S r-   )r   joinr   rJ   rC   r1   r1   r2   rA   
  s    
zPoolThread.stopc                 C   s
   t | _d S r-   )	TERMINATEr   rI   r1   r1   r2   	terminate  s    zPoolThread.terminatec                 C   s
   t | _d S r-   )CLOSEr   rI   r1   r1   r2   r     s    zPoolThread.close)N)r]   r^   r_   rN   r   r   r   rA   r   r   rk   r1   r1   rf   r2   r     s   
r   c                       s$   e Zd Z fddZdd Z  ZS )
Supervisorc                    s   || _ tt|   d S r-   )poolre   r   rN   )rJ   r   rf   r1   r2   rN     s    zSupervisor.__init__c                 C   s   t d td | j}z~|j}td|j d|_tdD ]*}| jtkr:|jtkr:|	  td q:||_| jtkr|jtkr|	  td qlW n& t
k
r   |  |   Y nX t d d S )Nzworker handler startingg?
   r   r,   zworker handler exiting)r!   r   r   r   r   
_processesr   r   r   _maintain_poolr   r   r   )rJ   r   Z
prev_stater   r1   r1   r2   r     s&    
zSupervisor.body)r]   r^   r_   rN   r   rk   r1   r1   rf   r2   r     s   r   c                       s4   e Zd Z fddZdd Zdd Zdd Z  ZS )	TaskHandlerc                    s0   || _ || _|| _|| _|| _tt|   d S r-   )	taskqueuer   outqueuer   cachere   r   rN   )rJ   r   r   r   r   r   rf   r1   r2   rN   >  s    zTaskHandler.__init__c           
      C   s  | j }| j}| j}t|jd D ]P\}}d }d}zt|D ]\}}| jrTtd  qz|| W q: tk
r   td Y  qY q: t	k
r   |d d \}}	z|| 
|	dt f W n tk
r   Y nX Y q:X q:|rtd ||d  W qW  qzW q t	k
rn   |r"|d d nd\}}	||krN|| 
|	d dt f |rjtd ||d  Y qX qtd	 |   d S )
Nz'task handler found thread._state != RUNzcould not put task on queuer'   Fzdoing set_length()r   )r   r   ztask handler got sentinel)r   r   r   iterr   	enumerater   r!   r   r   _setr   KeyErrorr   tell_others)
rJ   r   r   r   ZtaskseqZ
set_lengthtaskr   r   indr1   r1   r2   r   F  sH    


zTaskHandler.bodyc                 C   sl   | j }| j}| j}z0td |d  td |D ]}|d  q2W n tk
r^   td Y nX td d S )Nz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got IOError when sending sentinelsztask handler exiting)r   r   r   r!   r   )rJ   r   r   r   pr1   r1   r2   r  p  s    
zTaskHandler.tell_othersc                 C   s   |    d S r-   )r  rI   r1   r1   r2   r     s    zTaskHandler.on_stop_not_started)r]   r^   r_   rN   r   r  r   rk   r1   r1   rf   r2   r   <  s   *r   c                       sT   e Zd Z fddZdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
  ZS )TimeoutHandlerc                    s0   || _ || _|| _|| _d | _tt|   d S r-   )	processesr   t_softt_hard_itre   r  rN   )rJ   r  r   r  r	  rf   r1   r2   rN     s    zTimeoutHandler.__init__c                    s   t  fddt| jD dS )Nc                 3   s$   | ]\}}|j  kr||fV  qd S r-   r   .0r   procr   r1   r2   	<genexpr>  s   
z1TimeoutHandler._process_by_pid.<locals>.<genexpr>NN)nextr   r  r   r1   r   r2   _process_by_pid  s
    zTimeoutHandler._process_by_pidc              
   C   sv   t d| | |j\}}|s"d S |jdd zt|jt W n2 tk
rp } zt|tj	kr` W 5 d }~X Y nX d S )Nzsoft time limit exceeded for %rTsoft)
r!   r  _worker_pidhandle_timeoutr   r   OSErrorr   r   ESRCH)rJ   r   process_indexrd   r1   r1   r2   on_soft_timeout  s    
zTimeoutHandler.on_soft_timeoutc                 C   s|   |  rd S td| zt|jW n( tk
rL   ||jdt f Y nX | |j\}}|j	dd |rx| 
| d S )Nzhard time limit exceeded for %rFr  )r   r!   r   _timeoutr   _jobr   r  r  r  _trywaitkill)rJ   r   r  r  r1   r1   r2   on_hard_timeout  s    
zTimeoutHandler.on_hard_timeoutc                 C   s   t d|j zBt|j|jkrDt d|j tt|jtj n|  W n t	k
rb   Y nX |j
jddrvd S t d|j zFt|j|jkrt d|j tt|jtj nt|jt W n t	k
r   Y nX d S )Nztimeout: sending TERM to %szIworker %s is a group leader. It is safe to kill (SIGTERM) the whole groupr,   r   z/timeout: TERM timed-out, now sending KILL to %szIworker %s is a group leader. It is safe to kill (SIGKILL) the whole group)r!   _namer   getpgidr   killpgr   SIGTERMr   r  _popenwaitSIGKILLr   rJ   workerr1   r1   r2   r    s$    zTimeoutHandler._trywaitkillc                 #   s   | j | j }}t }| j}| j}dd }| jtkrt| j |rXt fdd|D } 	 D ]j\}}|j
}	|j}
|
d kr|}
|j}|d kr|}||	|r|| q`||kr`||	|
r`|| || q`d V  q(d S )Nc                 S   s"   | r|sdS t  | | krdS d S r   )r   )r   rC   r1   r1   r2   
_timed_out  s    z2TimeoutHandler.handle_timeouts.<locals>._timed_outc                 3   s   | ]}| kr|V  qd S r-   r1   )r  kr   r1   r2   r    s      z1TimeoutHandler.handle_timeouts.<locals>.<genexpr>)r	  r  setr  r  r   r   copyr   items_time_accepted_soft_timeoutr  add)rJ   r	  r  Zdirtyr  r  r(  r   r   Zack_timesoft_timeoutZhard_timeoutr1   r*  r2   handle_timeouts  s.    


zTimeoutHandler.handle_timeoutsc                 C   sN   | j tkrBz|  D ]}td qW q  tk
r>   Y qBY q X q td d S )Nr   ztimeout handler exiting)r   r   r2  r   r   r   r!   rJ   r   r1   r1   r2   r     s    
zTimeoutHandler.bodyc                 G   sB   | j d kr|  | _ zt| j  W n tk
r<   d | _ Y nX d S r-   )r
  r2  r  StopIterationrJ   r7   r1   r1   r2   handle_event  s    

zTimeoutHandler.handle_event)r]   r^   r_   rN   r  r  r  r  r2  r   r6  rk   r1   r1   rf   r2   r    s   &	r  c                       sT   e Zd Zd fdd	Zdd Zdd Zdd	d
ZdddZdd ZdddZ	  Z
S )ResultHandlerNc                    sb   || _ || _|| _|| _|| _|| _|| _d | _d| _|| _	|	| _
|
| _|   tt|   d S )NF)r   r   r   r   join_exited_workersputlockr   r
  _shutdown_completecheck_timeoutson_job_readyon_ready_counters_make_methodsre   r7  rN   )rJ   r   r   r   r   r8  r9  r   r;  r<  r=  rf   r1   r2   rN     s    zResultHandler.__init__c                 C   s   | j dd d S )NT)r2  )finish_at_shutdownrI   r1   r1   r2   r     s    z!ResultHandler.on_stop_not_startedc                    sl   j  jjj fdd} fdd}dd }t|t|t|i _fdd}|_d S )	Nc              	      s<   d_ z |  |||| W n ttfk
r6   Y nX d S rq   )R_ackr  r.   )r   r   time_acceptedr   r   )r   r   r1   r2   on_ack(  s
    z+ResultHandler._make_methods.<locals>.on_ackc              	      s   d k	r| ||| z |  }W n t k
r8   Y d S X jrtt| d }|r|jkrj| }|  | jd7  _W 5 Q R X | sd k	r  z|	|| W n t k
r   Y nX d S rF   )
r  r=  r  r   worker_pidsZget_lockrP   r   rX   r   )r   r   r   r   itemZ
worker_pidr   )r   r<  r9  rJ   r1   r2   on_ready0  s&    

z-ResultHandler._make_methods.<locals>.on_readyc              
   S   sH   zt | t W n2 tk
rB } zt|tjkr2 W 5 d }~X Y nX d S r-   )r   r&   r	   r  r   r   r  )r   r   rd   r1   r1   r2   on_deathG  s
    z-ResultHandler._make_methods.<locals>.on_deathc                    s>   | \}}z | |  W n  t k
r8   td|| Y nX d S )NzUnknown job state: %s (args=%s))r  r!   )r  stater7   )state_handlersr1   r2   on_state_changeR  s
    z4ResultHandler._make_methods.<locals>.on_state_change)	r   r9  r   r<  r   r   r   rI  rJ  )rJ   rC  rF  rG  rJ  r1   )r   r<  r9  r   rJ   rI  r2   r>  "  s          
zResultHandler._make_methodsr   c              
   c   s   | j }| j}z||\}}W n6 ttfk
rR } ztd| t W 5 d }~X Y nX | jrv| jtkshttd t |r|d krtd t || |dkrqnqd V  qd S )N result handler got %r -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelr   )	r   rJ  r   r   r!   r   r   r   rt   )rJ   rC   r   rJ  r   r  rd   r1   r1   r2   _process_resultZ  s(    
zResultHandler._process_resultc              	   C   sR   | j tkrN| jd kr | d| _zt| j W n ttfk
rL   d | _Y nX d S rq   )r   r   r
  rL  r  r4  r   )rJ   r/   eventsr1   r1   r2   r6  u  s    

zResultHandler.handle_eventc                 C   sV   t d z>| jtkrDz| dD ]}q W q
 tk
r@   Y qDY q
X q
W 5 |   X d S )Nzresult handler startingr   )r!   r?  r   r   rL  r   r3  r1   r1   r2   r   ~  s    
zResultHandler.bodyFc              
   C   s  d| _ | j}| j}| j}| j}| j}| j}| j}d }	|r*| jt	kr*|d k	rT|  z|d\}
}W n6 t
tfk
r } ztd| W Y d S d }~X Y nX |
r|d krtd q4|| z|dd W q4 tk
r&   t }|	s|}	n8||	 dkrtd Y q*tdtt||	 d d	 Y q4X q4t|d
rtd z,tdD ]}|j s^ qh|  qHW n t
tfk
r   Y nX tdt|| j d S )NTr   rK  z&result handler ignoring extra sentinel)shutdowng      @z!result handler exiting: timed outz6result handler: all workers terminated, timeout in %ssr   r   z"ensuring that outqueue is not fullr   z7result handler exiting: len(cache)=%s, thread._state=%s)r:  r   r   r   r   r8  r;  rJ  r   r   r   r   r!   rl   r   absminr   r   r   len)rJ   r2  r   r   r   r   r8  r;  rJ  Ztime_terminater   r  rd   r   r   r1   r1   r2   r?    s^    
 z ResultHandler.finish_at_shutdown)N)r   )NN)F)r]   r^   r_   rN   r   r>  rL  r6  r   r?  rk   r1   r1   rf   r2   r7  
  s    8

	r7  c                   @   s<  e Zd ZdZdZeZeZeZeZe	Z	e
Z
dwddZd	d
 Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd  Zdxd!d"Zd#d$ Zd%d& Zd'd( Zd)d* Zd+d, Zd-d. Zd/d0 Zd1d2 Z d3d4 Z!dyd5d6Z"dzd7d8Z#d9d: Z$d;d< Z%d=d> Z&d?d@ Z'dAdB Z(dCdD Z)dEdF Z*dGdH Z+dIdJ Z,di fdKdLZ-d{dMdNZ.d|dOdPZ/d}dQdRZ0d~dSdTZ1ddUdVZ2di ddddddddddfdWdXZ3dYdZ Z4dd[d\Z5dd]d^Z6dd_d`Z7e8dadb Z9dcdd Z:dedf Z;dgdh Z<e8didj Z=dkdl Z>dmdn Z?e8dodp Z@eAdqdr ZBeAdsdt ZCeDdudv ZEdS )PoolzS
    Class which supports an async version of applying functions to arguments.
    TNr1   r   Fc                 K   s  |pt  | _|| _|   t | _i | _t| _|| _	|| _
|| _|| _|| _|| _|| _|p^t| _|
| _|| _|| _|| _|| _i | _|| _t|p| j	d k	p| j
d k	| _|rtd krttd d }|d kr|  n|| _ |pt!| j d | _"t#||	pd| _#|d k	rt$|st%d|d k	r:t$|s:t%d| jj&| _'g | _(i | _)i | _*|| _+|pjt,| j | _-t.| j D ]}| /| qx| 0| | _1|r| j12  | 3| j| j4| j5| j(| j| _6|r| j62  d | _7| jr$| 8| j(| j| j
| j	| _9t: | _;d| _<| =  |s6| j9j>| _7nd | _9d| _<d | _;| ? | _@| j@j>| _A|rZ| j@2  tB| | jC| j| jD| j5| j(| j1| j6| j@| j| j9| E f
dd| _Fd S )	NWSoft timeouts are not supported: on this platform: It does not have the SIGUSR1 signal.d   r   zinitializer must be a callablez on_process_exit must be callableF   )r7   Zexitpriority)Gr   _ctxsynack_setup_queuesr   
_taskqueue_cacher   r   rC   r1  _maxtasksperchild_max_memory_per_child_initializer	_initargs_on_process_exitLOST_WORKER_TIMEOUTlost_worker_timeouton_process_upon_process_downon_timeout_seton_timeout_cancelthreadsZreadersallow_restartboolenable_timeoutsr   warningswarnUserWarningr   r   roundmax_restartsr   callable	TypeErrorProcess_Process_pool	_poolctrl_on_ready_countersputlocksrE   _putlockr   _create_worker_processr   _worker_handlerr   r   r   	_outqueue_task_handlerr;  r  _timeout_handlerLock_timeout_handler_mutex_timeout_handler_started_start_timeout_handlerr6  create_result_handler_result_handlerhandle_result_eventr    _terminate_pool_inqueue_help_stuff_finish_args
_terminate)rJ   r  ru   rv   ZmaxtasksperchildrC   r1  ra  rn  Zmax_restart_freqrb  rc  rd  re  rf  Z	semaphorerv  rg  rW  Zon_process_exitcontextrx   ri  r>   r   r1   r1   r2   rN     s    



  


    zPool.__init__c                 O   s   | j ||S r-   )rr  )rJ   r7   kwdsr1   r1   r2   rq  I  s    zPool.Processc                 C   s   | | j|dS )N)target)r   rq  r&  r1   r1   r2   WorkerProcessL  s    zPool.WorkerProcessc              
   K   s:   | j | j| j| j| j| j| j| j| j| j	f	d| j
i|S )Nr=  )r7  rz  r   rZ  _poll_result_join_exited_workersrw  r   r;  r<  ru  )rJ   extra_kwargsr1   r1   r2   r  O  s         zPool.create_result_handlerc                 C   s   d S r-   r1   )rJ   r   r   r   r   r1   r1   r2   r<  X  s    zPool.on_job_readyc                 C   s   | j | j| jfS r-   )r  r{  rs  rI   r1   r1   r2   r  [  s    zPool._help_stuff_finish_argsc                 C   s$   zt  W S  tk
r   Y dS X d S rF   )r   NotImplementedErrorrI   r1   r1   r2   r   ^  s    zPool.cpu_countc                 G   s   | j j| S r-   )r  r6  r5  r1   r1   r2   r  d  s    zPool.handle_result_eventc                 C   s   d S r-   r1   )rJ   r'  Zqueuesr1   r1   r2   _process_register_queuesg  s    zPool._process_register_queuesc                    s   t  fddt| jD dS )Nc                 3   s$   | ]\}}|j  kr||fV  qd S r-   r   r  r   r1   r2   r  k  s   
z'Pool._process_by_pid.<locals>.<genexpr>r  )r  r   rs  r   r1   r   r2   r  j  s
    zPool._process_by_pidc                 C   s   | j | jd fS r-   )r  rz  rI   r1   r1   r2   get_process_queuesp  s    zPool.get_process_queuesc                 C   s   | j r| j nd }|  \}}}| jd}| | j|||| j| j| j	|| j
| j| j| j|d}| j| | ||||f |jdd|_d|_||_|  || j|j< || j|j< | jr| | |S )Nr   )r{   r   rx   r   rq  Z
PoolWorkerT)rg  rV  Eventr  Valuer  rp   r]  r^  r[  r_  rf  _wrap_exceptionr\  rs  appendr  namereplacer   indexr   rt  r   ru  rb  )rJ   r   r   r|   r}   r~   r   wr1   r1   r2   rx  s  s8          

zPool._create_worker_processc                 C   s   d S r-   r1   r&  r1   r1   r2   process_flush_queues  s    zPool.process_flush_queuesc                    s|  d}dd t | j D D ]2}|p(t }|j\}}|| |jkr| || q|rdt| jsdt	 i i  }t
tt| jD ]}| j| }|j}	|j}
|
dks|	dk	rtd| |
dk	r|  td| ||j< |	||j< |	ttfkrt|ddstd|j|jt|	d	d
 | | | j|= | j|j= | j|j= qrxdd | jD  t | j D ]}t fdd| D d}|r| || | s6||pd	}	|}|rt|ddr||	 n|  |||	 nJ|j!}|j"}|r|# s| ||j n|rd|# sd| ||j qdtD ](}| j$rB|s^| %| | $| qBt | S g S )zCleanup after any worker processes which have exited due to
        reaching their specified lifetime. Returns True if any workers were
        cleaned up.
        Nc                 S   s   g | ]}|  s|jr|qS r1   )r   _worker_lost)r  r   r1   r1   r2   
<listcomp>  s     z-Pool._join_exited_workers.<locals>.<listcomp>z!Supervisor: cleaning up worker %dzSupervisor: worked %d joined_controlled_terminationFz Process %r pid:%r exited with %rr   r   c                 S   s   g | ]
}|j qS r1   r   r  r  r1   r1   r2   r    s     c                 3   s"   | ]}|ks| kr|V  qd S r-   r1   r  r   Zall_pidscleanedr1   r2   r    s     z,Pool._join_exited_workers.<locals>.<genexpr>_job_terminated)&r4   rZ  r   r   r  _lost_worker_timeoutmark_as_worker_lostrQ  rs  rl   reversedr   r   r#  r!   r   r   r+   r   getattrr<   r  r
   r  rt  ru  r  rD  on_job_process_downr   r   _set_terminatedon_job_process_lost	_write_to_scheduled_for	_is_aliverc  _process_cleanup_queues)rJ   rN  r   r   Z	lost_timeZlost_ret	exitcodesr   r'  r   popenZacked_by_goner  Zwrite_toZ	sched_forr1   r  r2   r    s    








  



  
zPool._join_exited_workersc                 C   s   d S r-   r1   )rJ   r   r'  r1   r1   r2   on_partial_read  s    zPool.on_partial_readc                 C   s   d S r-   r1   r&  r1   r1   r2   r    s    zPool._process_cleanup_queuesc                 C   s   d S r-   r1   )rJ   r   Zpid_goner1   r1   r2   r    s    zPool.on_job_process_downc                 C   s   t  |f|_d S r-   )r   r  )rJ   r   r   r   r1   r1   r2   r    s    zPool.on_job_process_lostc                 C   sF   zt dt||jW n& t k
r@   |d dt f Y nX d S )Nz(Worker exited prematurely: {0} Job: {1}.F)r   r   r
   r  r   r   )rJ   r   r   r1   r1   r2   r    s     zPool.mark_as_worker_lostc                 C   s   | S r-   r1   rI   r1   r1   r2   	__enter__  s    zPool.__enter__c                 G   s   |   S r-   )r   )rJ   r   r1   r1   r2   __exit__  s    zPool.__exit__c                 C   s   d S r-   r1   rJ   nr1   r1   r2   on_grow   s    zPool.on_growc                 C   s   d S r-   r1   r  r1   r1   r2   	on_shrink  s    zPool.on_shrinkc                 C   sb   t |  D ]H\}}|  jd8  _| jr2| j  |  | d ||d kr q^qtdd S )Nr   z&Can't shrink pool. All processes busy!)r   _iterinactiver   rw  rK   Zterminate_controlledr  
ValueError)rJ   r  r   r'  r1   r1   r2   rK     s    

zPool.shrinkc                 C   s:   t |D ]"}|  jd7  _| jr| j  q| | d S rF   )r   r   rw  rU   r  )rJ   r  r   r1   r1   r2   rU     s
    z	Pool.growc                 c   s    | j D ]}| |s|V  qd S r-   )rs  _worker_activer&  r1   r1   r2   r    s    

zPool._iterinactivec                 C   s(   t | jD ]}|j| kr
 dS q
dS )NTF)r   rZ  r   rD  )rJ   r'  r   r1   r1   r2   r    s    zPool._worker_activec              	   C   s   t | jt| j D ]l}| jtkr( dS z"|rH|| ttfkrH| j	  W n t
k
rh   | j	  Y nX | |   td qdS )zBring the number of pool processes up to the specified number,
        for use after reaping workers which have exited.
        Nzadded worker)r   r   rQ  rs  r   r   r+   r   r   step
IndexErrorrx  _avail_indexr!   )rJ   r  r   r1   r1   r2   _repopulate_pool$  s    
zPool._repopulate_poolc                    sD   t | j| jk sttdd | jD  t fddt| jD S )Nc                 s   s   | ]}|j V  qd S r-   )r  )r  r  r1   r1   r2   r  5  s     z$Pool._avail_index.<locals>.<genexpr>c                 3   s   | ]}| kr|V  qd S r-   r1   )r  r   indicesr1   r2   r  6  s      )rQ  rs  r   rt   r+  r  r   rI   r1   r  r2   r  3  s    zPool._avail_indexc                 C   s
   |    S r-   )r  rI   r1   r1   r2   did_start_ok8  s    zPool.did_start_okc                 C   s<   |   }| | tt|D ]}| jdk	r| j  qdS )zF"Clean up any exited workers and start replacements for them.
        N)r  r  r   rQ  rw  rX   )rJ   Zjoinedr   r1   r1   r2   r   ;  s
    

zPool._maintain_poolc              
   C   s   | j jtkr| jtkrz|   W nr tk
rH   |   |    Y nN tk
r } z0t|t	j
krtttt|t d   W 5 d }~X Y nX d S )Nr'   )ry  r   r   r   r   r   r   r  r   r   ZENOMEMr   MemoryErrorrh   r   r   r   r1   r1   r2   maintain_poolD  s    

zPool.maintain_poolc                    sF    j   _ j   _ jjj _ jjj _	 fdd}| _
d S )Nc                    s    j j| rd  fS dS r   )rz  r   r   r   r   rI   r1   r2   r  Y  s    z(Pool._setup_queues.<locals>._poll_result)rV  SimpleQueuer  rz  r   r   r   r   r   r   r  rJ   r  r1   rI   r2   rX  S  s    zPool._setup_queuesc              	   C   s<   | j r8| jd k	r8| j | js.d| _| j  W 5 Q R X d S r   )rf  r|  r~  r  r   rI   r1   r1   r2   r  _  s
    zPool._start_timeout_handlerc                 C   s    | j tkr| ||| S dS )z8
        Equivalent of `func(*args, **kwargs)`.
        N)r   r   apply_asyncr   )rJ   funcr7   r  r1   r1   r2   applyh  s    
z
Pool.applyc                 C   s"   | j tkr| ||t| S dS )z
        Like `map()` method but the elements of the `iterable` are expected to
        be iterables as well and will be unpacked as arguments. Hence
        `func` and (a, b) becomes func(a, b).
        N)r   r   
_map_asyncr;   r   rJ   r  iterable	chunksizer1   r1   r2   r:   o  s
    
 zPool.starmapc                 C   s"   | j tkr| ||t|||S dS )z=
        Asynchronous version of `starmap()` method.
        N)r   r   r  r;   rJ   r  r  r  callbackerror_callbackr1   r1   r2   starmap_asyncy  s
    
 zPool.starmap_asyncc                 C   s    | j tkr| ||| S dS )zx
        Apply `func` to each element in `iterable`, collecting the results
        in a list that is returned.
        N)r   r   	map_asyncr   r  r1   r1   r2   r5     s    
zPool.mapc                    s   | j tkrdS |p| j}|dkrXt| j|d| j fddt|D jf S |dksdt	t
 ||}t| j|d| jfddt|D jf dd D S dS )zP
        Equivalent of `map()` -- can be MUCH slower than `Pool.map()`.
        Nr   ra  c                 3   s(   | ] \}}t j| |fi ffV  qd S r-   r   r  r  r   xr  r   r1   r2   r    s   zPool.imap.<locals>.<genexpr>c                 3   s(   | ] \}}t  j|t|fi ffV  qd S r-   r   r  r8   r  r   r1   r2   r    s   c                 s   s   | ]}|D ]
}|V  q
qd S r-   r1   r  chunkrE  r1   r1   r2   r    s       )r   r   ra  IMapIteratorrZ  rY  r   r   _set_lengthrt   rR  
_get_tasksrJ   r  r  r  ra  task_batchesr1   r  r2   imap  s4    


z	Pool.imapc                    s   | j tkrdS |p| j}|dkrXt| j|d| j fddt|D jf S |dksdt	t
 ||}t| j|d| jfddt|D jf dd D S dS )zL
        Like `imap()` method but ordering of results is arbitrary.
        Nr   r  c                 3   s(   | ] \}}t j| |fi ffV  qd S r-   r  r  r  r1   r2   r    s   z&Pool.imap_unordered.<locals>.<genexpr>c                 3   s(   | ] \}}t  j|t|fi ffV  qd S r-   r  r  r  r1   r2   r    s   c                 s   s   | ]}|D ]
}|V  q
qd S r-   r1   r  r1   r1   r2   r    s       )r   r   ra  IMapUnorderedIteratorrZ  rY  r   r   r  rt   rR  r  r  r1   r  r2   imap_unordered  s8    

  
zPool.imap_unorderedc                 C   s  | j tkrdS |	p| j}	|
p | j}
|p*| j}|	rJtdkrJttd d}	| j tkr|dkrd| j	n|}|r| j
dk	r| j
  t| j|||||	|
|| j| j|| jr| jnd|d}|
s|	r|   | jr| jt|jd|||ffgdf n| t|jd|||ff |S dS )a  
        Asynchronous equivalent of `apply()` method.

        Callback is called when the functions return value is ready.
        The accept callback is called when the job is accepted to be executed.

        Simplified the flow is like this:

            >>> def apply_async(func, args, kwds, callback, accept_callback):
            ...     if accept_callback:
            ...         accept_callback()
            ...     retval = func(*args, **kwds)
            ...     if callback:
            ...         callback(retval)

        NrS  )rd  re  callbacks_propagatesend_ackcorrelation_id)r   r   r1  rC   ra  r   rj  rk  rl  rv  rw  rH   ApplyResultrZ  rd  re  rW  r  r  rf  rY  r   r   r  r   )rJ   r  r7   r  r  r  accept_callbacktimeout_callbackZwaitforslotr1  rC   ra  r  r  r   r1   r1   r2   r    sR    




      	  zPool.apply_asyncc                 C   s   d S r-   r1   )rJ   responser   r   fdr1   r1   r2   r    s    zPool.send_ackc              
   C   sl   |  |\}}|d k	rhzt||p"t W n2 tk
rZ } zt|tjkrJ W 5 d }~X Y nX d|_d|_d S r   )	r  r   r	   r  r   r   r  r  r  )rJ   r   sigr  r   rd   r1   r1   r2   terminate_job   s    zPool.terminate_jobc                 C   s   |  ||t|||S )z<
        Asynchronous equivalent of `map()` method.
        )r  r8   r  r1   r1   r2   r    s         zPool.map_asyncc           	         s   | j tkrdS t|ds t|}|dkrPtt|t| jd \}}|rP|d7 }t|dkr`d}t|||}t	| j
|t|||d| j fddt|D df S )	zY
        Helper function to implement map, starmap and their async counterparts.
        N__len__r(   r   r   r  c                 3   s(   | ] \}}t j| |fi ffV  qd S r-   r  r  mapperr   r1   r2   r  )  s   z"Pool._map_async.<locals>.<genexpr>)r   r   r   r4   divmodrQ  rs  rR  r  	MapResultrZ  rY  r   r   )	rJ   r  r  r  r  r  r  extrar  r1   r  r2   r    s(    

zPool._map_asyncc                 c   s0   t |}tt||}|s d S | |fV  qd S r-   )r   tupler9   islice)r  itsizer  r1   r1   r2   r  -  s
    zPool._get_tasksc                 C   s   t dd S )Nz:pool objects cannot be passed between processes or pickled)r  rI   r1   r1   r2   r   6  s    zPool.__reduce__c                 C   sL   t d | jtkrHt| _| jr(| j  | j  | j	d  t
| j d S )Nzclosing pool)r!   r   r   r   rw  rY   ry  r   rY  r   rD   rI   r1   r1   r2   r   ;  s    


z
Pool.closec                 C   s$   t d t| _| j  |   d S )Nzterminating pool)r!   r   r   ry  r   r  rI   r1   r1   r2   r   E  s    
zPool.terminatec                 C   s   t |  d S r-   )rD   )task_handlerr1   r1   r2   _stop_task_handlerK  s    zPool._stop_task_handlerc                 C   s   | j ttfksttd t| j td | | j td t| j	 td t
| jD ]2\}}td|d t| j| |jd k	r\|  q\td d S )Nzjoining worker handlerjoining task handlerjoining result handlerzresult handler joinedzjoining worker %s/%s (%r)r   zpool join complete)r   r   r   rt   r!   rD   ry  r  r{  r  r   rs  rQ  r#  r   )rJ   r   r  r1   r1   r2   r   O  s    



z	Pool.joinc                 C   s   t | jD ]}|  q
d S r-   )r   rt  r+  )rJ   er1   r1   r2   restart^  s    zPool.restartc                 C   s>   t d | j  | r:| j r:| j  td qd S )Nz7removing tasks from inqueue until task handler finishedr   )	r!   Z_rlockrH   is_aliver   r   r   r   r   )inqueuer  rs  r1   r1   r2   _help_stuff_finishb  s
    

zPool._help_stuff_finishc                 C   s   | d  d S r-   )r   )clsr   r   r1   r1   r2   _set_result_sentinelk  s    zPool._set_result_sentinelc                 C   s>  t d |  |  |d  t d | j|
  |  | || |	d k	rX|	  |rt|d drt d |D ]}| rv|  qvt d | | t d |  |	d k	rt d |	t	 |rt|d drt d	 |D ]*}|
 rt d
|j |jd k	r|  qt d |r,|  |r:|  d S )Nzfinalizing poolz&helping task handler/workers to finishr   r   zterminating workersr  r  zjoining timeout handlerzjoining pool workerszcleaning up worker %dzpool workers joined)r!   r   r   r  r  r   r  r  rA   TIMEOUT_MAXr  r   r#  r   r   )r   r   r  r   r   Zworker_handlerr  Zresult_handlerr   Ztimeout_handlerZhelp_stuff_finish_argsr  r1   r1   r2   r  o  sD    






zPool._terminate_poolc                 C   s   dd | j D S )Nc                 S   s   g | ]}|j jqS r1   )r#  r   r  r1   r1   r2   r    s     z*Pool.process_sentinels.<locals>.<listcomp>)rs  rI   r1   r1   r2   process_sentinels  s    zPool.process_sentinels)NNr1   NNNNNr   NNNNTNFFFNNNF)F)r   )r   )N)NNN)N)r   N)r   N)N)NNN)NNN)Fr]   r^   r_   r`   r  rp   r   r   r  r7  r   rN   rq  r  r  r<  r  r   r  r  r  r  rx  r  r  r  r  r  r  r  r  r  r  r  rK   rU   r  r  r  r  r  r   r  rX  r  r  r:   r  r5   r  r  r  r  r  r  r  staticmethodr  r   r   r   r  r   r  r  classmethodr  r  propertyr  r1   r1   r1   r2   rR    s                          
z	
S

		

    
	

  
     
8
    
	    






6rR  c                   @   s   e Zd ZdZdZdZdddddedddddfddZdd Zdd Z	d	d
 Z
dd Zdd Zdd Zdd Zd$ddZdd Zd%ddZd&ddZdd Zd'ddZd d! Zd"d# ZdS )(r  Nr1   c                 C   s   || _ t | _t | _tt| _|| _	|| _
|| _|| _|| _|| _|| _|| _|	| _|
| _|pdd| _|| _d| _d| _d | _d | _d | _| || j< d S )Nr1   F)r  r}  _mutexr?   r  _eventr  job_counterr  rZ  	_callback_accept_callback_error_callback_timeout_callbackr  r/  r  _on_timeout_set_on_timeout_cancel_callbacks_propagate	_send_ack	_accepted
_cancelledr  r.  _terminated)rJ   r   r  r  r  r  r1  rC   ra  rd  re  r  r  r  r1   r1   r2   rN     s,    


zApplyResult.__init__c                 C   s   dj | jj| j| j|  dS )Nz"<%s: {id} ack:{ack} ready:{ready}>)idZackr   )r   rg   r]   r  r  r   rI   r1   r1   r2   ri     s      zApplyResult.__repr__c                 C   s
   | j  S r-   )r  isSetrI   r1   r1   r2   r     s    zApplyResult.readyc                 C   s   | j S r-   )r  rI   r1   r1   r2   accepted  s    zApplyResult.acceptedc                 C   s   |   st| jS r-   )r   rt   _successrI   r1   r1   r2   
successful  s    zApplyResult.successfulc                 C   s
   d| _ dS )zOnly works if synack is used.TN)r  rI   r1   r1   r2   _cancel  s    zApplyResult._cancelc                 C   s   | j | jd  d S r-   )rZ  popr  rI   r1   r1   r2   discard  s    zApplyResult.discardc                 C   s
   || _ d S r-   )r  rJ   rm   r1   r1   r2   r     s    zApplyResult.terminatec                 C   s>   zt |p
d W n& t k
r8   | d dt f Y nX d S Nr   F)r   r   r   r  r1   r1   r2   r    s    zApplyResult._set_terminatedc                 C   s   | j r| j gS g S r-   r  rI   r1   r1   r2   rD    s    zApplyResult.worker_pidsc                 C   s   | j | d S r-   )r  r$  r   r1   r1   r2   r$    s    zApplyResult.waitc                 C   s.   |  | |  st| jr"| jS | jjd S r-   )r$  r   r   r  rS   	exceptionr   r1   r1   r2   r     s    
zApplyResult.getc              
   O   s^   |rZz||| W nF | j k
r*    Y n0 tk
rX } ztd|dd W 5 d }~X Y nX d S )Nz"Pool callback raised exception: %rr   r   )r  r   r<   )rJ   r   r7   r>   rd   r1   r1   r2   safe_apply_callback  s    zApplyResult.safe_apply_callbackFc                 C   s,   | j d k	r(| j| j ||r| jn| jd d S )N)r  rC   )r  r!  r/  r  )rJ   r  r1   r1   r2   r    s    
 zApplyResult.handle_timeoutc              	   C   s   | j  | jr| |  |\| _| _| j  | jrD| j| j	d  | j
r`| jr`| | j
| j | jd k	r| jr| js| | j| j W 5 Q R X d S r-   )r  r  r  rS   r  r+  r  rZ  r  r  r
  r!  r  rJ   r   r   r1   r1   r2   r     s*    

 
 zApplyResult._setc                 C   sJ  | j 8 | jrL| jrLd| _|r>| t|| j|W  5 Q R  S W 5 Q R  d S d| _|| _|| _|  rv| j	
| jd  | jr| | | j| j t}| jrd zHz| || W n2 | jk
r   t} Y n tk
r   t}Y nX W 5 | jr|r| ||| j|  W  5 Q R  S X  | jr<|r<| ||| j| W 5 Q R X d S r   )r  r  r  r  r   r  r.  r  r   rZ  r  r  r/  r  r   r  Z_propagate_errorsr   )rJ   r   rB  r   r   r  r1   r1   r2   rA  '  s>    
   zApplyResult._ack)N)N)N)F)r]   r^   r_   r  r  r  r`  rN   ri   r   r  r  r  r  r   r  rD  r$  r   r!  r  r   rA  r1   r1   r1   r2   r    s<        



	

r  c                   @   s4   e Zd Zdd Zdd Zdd Zdd Zd	d
 ZdS )r  c                 C   s   t j| |||d d| _|| _d g| | _dg| | _d g| | _d g| | _|| _|dkrvd| _	| j
  || j= n|| t||  | _	d S )Nr  TFr   )r  rN   r  _lengthrS   r  r  r.  
_chunksize_number_leftr  r+  r  rh  )rJ   r   r  lengthr  r  r1   r1   r2   rN   O  s$       

zMapResult.__init__c                 C   s   |\}}|rv|| j || j |d | j < |  jd8  _| jdkr| jrT| | j  | jrj| j| jd  | j	  n>d| _
|| _ | jr| | j  | jr| j| jd  | j	  d S )Nr   r   F)rS   r$  r%  r
  r  rZ  r  r  r  r+  r  r  )rJ   r   Zsuccess_resultsuccessr   r1   r1   r2   r   a  s"    
zMapResult._setc                 G   sj   || j  }t|d | j  | j}t||D ]"}d| j|< || j|< || j|< q*|  rf| j	| j
d  d S Nr   T)r$  rP  r#  r   r  r  r.  r   rZ  r  r  )rJ   r   rB  r   r7   r   rA   jr1   r1   r2   rA  u  s    


zMapResult._ackc                 C   s
   t | jS r-   )allr  rI   r1   r1   r2   r    s    zMapResult.acceptedc                 C   s   dd | j D S )Nc                 S   s   g | ]}|r|qS r1   r1   r  r1   r1   r2   r    s      z)MapResult.worker_pids.<locals>.<listcomp>r  rI   r1   r1   r2   rD    s    zMapResult.worker_pidsN)r]   r^   r_   rN   r   rA  r  rD  r1   r1   r1   r2   r  M  s
   
r  c                   @   sZ   e Zd ZdZefddZdd ZdddZeZdd	 Z	d
d Z
dd Zdd Zdd ZdS )r  Nc                 C   sZ   t t  | _tt| _|| _t | _	d| _
d | _d| _i | _g | _|| _| || j< d S r  )r?   	Conditionr}  rR   r  r	  r  rZ  r   _itemsr  r#  _ready	_unsorted_worker_pidsr  )rJ   r   ra  r1   r1   r2   rN     s    
zIMapIterator.__init__c                 C   s   | S r-   r1   rI   r1   r1   r2   __iter__  s    zIMapIterator.__iter__c                 C   s   | j  z| j }W nt tk
r   | j| jkr<d| _t| j | z| j }W n. tk
r   | j| jkr|d| _tt	Y nX Y nX W 5 Q R X |\}}|r|S t
|d S r   )rR   r,  popleftr  r  r#  r-  r4  r$  r   r   )rJ   rC   rE  r'  rP   r1   r1   r2   r    s&    zIMapIterator.nextc              	   C   s   | j  | j|krn| j| |  jd7  _| j| jkrb| j| j}| j| |  jd7  _q,| j   n
|| j|< | j| jkrd| _| j	| j
= W 5 Q R X d S r(  )rR   r  r,  r  r.  r  rT   r#  r-  rZ  r  r"  r1   r1   r2   r     s    

zIMapIterator._setc              	   C   sB   | j 2 || _| j| jkr4d| _| j   | j| j= W 5 Q R X d S r   )rR   r#  r  r-  rT   rZ  r  )rJ   r&  r1   r1   r2   r    s    
zIMapIterator._set_lengthc                 G   s   | j | d S r-   )r/  r  )rJ   r   rB  r   r7   r1   r1   r2   rA    s    zIMapIterator._ackc                 C   s   | j S r-   )r-  rI   r1   r1   r2   r     s    zIMapIterator.readyc                 C   s   | j S r-   )r/  rI   r1   r1   r2   rD    s    zIMapIterator.worker_pids)N)r]   r^   r_   r  r`  rN   r0  r  __next__r   r  rA  r   rD  r1   r1   r1   r2   r    s   
r  c                   @   s   e Zd Zdd ZdS )r  c              	   C   sV   | j F | j| |  jd7  _| j   | j| jkrHd| _| j| j= W 5 Q R X d S r(  )	rR   r,  r  r  rT   r#  r-  rZ  r  r"  r1   r1   r2   r     s    
zIMapUnorderedIterator._setN)r]   r^   r_   r   r1   r1   r1   r2   r    s   r  c                   @   s:   e Zd ZddlmZ eZdddZdd Zed	d
 Z	dS )
ThreadPoolr   )rq  Nr1   c                 C   s   t | ||| d S r-   )rR  rN   )rJ   r  ru   rv   r1   r1   r2   rN     s    zThreadPool.__init__c                    s:   t   _t   _ jj _ jj _ fdd}| _d S )Nc                    s.   zd j | dfW S  tk
r(   Y dS X d S r   )r   r   r   rI   r1   r2   r    s    z.ThreadPool._setup_queues.<locals>._poll_result)r   r  rz  r   r   r   r   r  r  r1   rI   r2   rX    s    

zThreadPool._setup_queuesc              	   C   s@   | j 0 | j  | jd gt|  | j   W 5 Q R X d S r-   )	not_emptyqueuerY   extendrQ  rV   )r  r  r   r1   r1   r2   r    s    
zThreadPool._help_stuff_finish)NNr1   )
r]   r^   r_   dummyrq  r   rN   rX  r  r  r1   r1   r1   r2   r3    s   
r3  )N)k
__future__r   r,  r   r9   r   r   r   r   r?   r   rj  collectionsr   	functoolsr    r   r   r   commonr	   r
   r   r   r   compatr   r   r   r   r   r7  r   
exceptionsr   r   r   r   r   r   r   Zfiver   r   r   r   r   r   r    r!   r"   r   version_infora   systemZ_winr%   r   r%  r&   r  r.   	SemaphorerM   r   r   r   r   r   r   r   r   r+   r   r   r  r   r`  r   r   countr	  r}  r3   r8   r;   r<   rD   rE   r   rb   rl   ro   objectrp   r   r   r   r  r7  rR  r  r  r  r  r3  r1   r1   r1   r2   <module>
   s   $	 



;  )%K  :     q =R