U
    .dTF                     @   s"  d dl mZmZmZ d dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ ddlmZmZmZ ddlmZmZ dd	lmZ ed
ZedZd dlmZmZmZ d dlm
Z
 d dlmZmZ dd ZdVddZ dd Z!dd Z"dd Z#dd Z$dd Z%ej&'ddd gd!d" Z(d#d$ Z)G d%d& d&e*Z+dWd'd(Z,d)d* Z-d+d, Z.ej&'d-e/e0dd.d/ Z1d0d1 Z2d2d3 Z3d4d5 Z4d6d7 Z5d8d9 Z6d:d; Z7d<d= Z8d>d? Z9d@dA Z:dBdC Z;dDdE Z<ej&'dFdGdHgej&j=ej>dIkoej>dJkdKdLdMdN Z?dOdP Z@ej&'dQdRdSgdTdU ZAdS )X    )print_functiondivisionabsolute_importNrandom)uuid4)sleep   )Paralleldelayedparallel_backend)ThreadingBackendAutoBatchingMixin)DaskDistributedBackenddistributeddask)ClientLocalCluster
get_client)time)clusterincc                  O   s   d S N )argskwargsr   r   9/tmp/pip-unpacked-wheel-ho8k9mhl/joblib/test/test_dask.pynoop   s    r   皙?c                 C   s   t | | rtdd S )Nzcondition evaluated to True)r   
ValueError)	conditiondurationr   r   r   slow_raise_value_error   s    r"   c                    sF   | dd }i }| D ]&\}}t fddt|D ||< q|S )Nc                 S   s   | j S r   )log)Zdask_workerr   r   r   <lambda>        zcount_events.<locals>.<lambda>c                    s   g | ]}|d   kr|qS    r   ).0event
event_namer   r   
<listcomp>#   s    z count_events.<locals>.<listcomp>)runitemslenlist)r+   clientZworker_eventsZevent_countsweventsr   r*   r   count_events   s
     r4   c                 C   s   t  \}\}}t|d | d}td\}}t dd tdD }|dd tdD ksbttt t d	d tdD  W 5 Q R X t d
d tdD }|dd tdD kstW 5 Q R X W 5 Q R X W 5 Q R X d S )Naddressloopr   c                 s   s   | ]}t t|V  qd S r   r   r   r(   ir   r   r   	<genexpr>,   s     ztest_simple.<locals>.<genexpr>
   c                 S   s   g | ]}t |qS r   r   r9   r   r   r   r,   -   s     ztest_simple.<locals>.<listcomp>c                 s   s   | ]}t t|d kV  qdS )   N)r   r"   r9   r   r   r   r;   0   s   c                 s   s   | ]}t t|V  qd S r   r8   r9   r   r   r   r;   3   s     c                 S   s   g | ]}t |qS r   r=   r9   r   r   r   r,   4   s     )	r   r   r   r
   rangeAssertionErrorpytestraisesr   r7   sabr1   ba_seqr   r   r   test_simple(   s    
rJ   c           	      C   s   t jtjkstt \}\}}t|d | d}tdt\}}t ^}|j}t	|t s\t|j
|ksjt|jdksxt|dd ttdD  |jdkstW 5 Q R X W 5 Q R X W 5 Q R X W 5 Q R X d S )	Nr5   r6   r   r'   c                 s   s   | ]}t d d  V  qdS )c                   S   s   d S r   r   r   r   r   r   r$   I   r%   z?test_dask_backend_uses_autobatching.<locals>.<genexpr>.<lambda>Nr   r(   rH   r   r   r   r;   H   s   z6test_dask_backend_uses_autobatching.<locals>.<genexpr>     @r<   )r   Zcompute_batch_sizer   r@   r   r   r   r
   _backend
isinstanceparallelZ_effective_batch_sizer?   int)	r7   rD   rE   rF   r1   rG   rH   rP   backendr   r   r   #test_dask_backend_uses_autobatching7   s    
rS   c                   C   s   t  S r   r   r   r   r   r   random2O   s    rT   c           	      C   sz   t  j\}\}}t|d | dF}td2\}}t dd tdD \}}||ksXtW 5 Q R X W 5 Q R X W 5 Q R X d S )Nr5   r6   r   c                 s   s   | ]}t t V  qd S r   )r   rT   r9   r   r   r   r;   W   s     z3test_dont_assume_function_purity.<locals>.<genexpr>r	   r   r   r   r
   r?   r@   )	r7   rD   rE   rF   r1   rG   rH   xyr   r   r    test_dont_assume_function_purityS   s
    rX   mixedTFc                 C   s   ddl m} |s(dd tdD }d}ndd tdD }d}t|||ksRtt \}\}}t|d	 | d
b}td\}	}
tddd|}
W 5 Q R X dd }|	dd}|
|}tdd |D stW 5 Q R X W 5 Q R X d S )Nr   )Batchc                 S   s   g | ]}t t|qS r   r8   r9   r   r   r   r,   _   s     z&test_dask_funcname.<locals>.<listcomp>   Zbatch_of_inc_4_callsc                 S   s,   g | ]$}|d  rt t|n
t t|qS )r	   )r   absr   r9   r   r   r   r,   b   s    Zmixed_batch_of_inc_4_callsr5   r6   r   r	   all)Z
batch_sizeZpre_dispatchc                 S   s
   t | jS r   )r0   Ztransition_log)Zdask_schedulerr   r   r   fn   s    ztest_dask_funcname.<locals>.f42c                 s   s   | ]}d |d kV  qdS )Zbatch_of_incr   Nr   )r(   tupr   r   r   r;   r   s     z%test_dask_funcname.<locals>.<genexpr>)Zjoblib._daskrZ   r?   reprr@   r   r   r   r
   replaceZrun_on_schedulerr]   )r7   rY   rZ   tasksZ
batch_reprrD   rE   rF   r1   rG   rH   r^   r#   r   r   r   test_dask_funcname[   s"    
re   c              	      sR  dd t dD }tdtd d fdd	tdd	d
}t|}ztd"\}}t	 fdd|D }W 5 Q R X |dd t dD kst
td|}t| dkst
tdd |D st
td$\}}t	  fdd|D }W 5 Q R X td|}t| dks t
tdd |D s8t
W 5 |  |  X d S )Nc                 S   s   g | ]}g qS r   r   rL   r   r   r   r,      s     z;test_no_undesired_distributed_cache_hit.<locals>.<listcomp>d   numpyg    .Ac                    s(   |d k	rj |  | t j | S r   )ZtestingZassert_array_equalappendr   hex)list_data)Xnpr   r   isolated_operation   s    zCtest_no_undesired_distributed_cache_hit.<locals>.isolated_operationr'   r	   )	n_workersthreads_per_workerr   c                 3   s   | ]}t  |V  qd S r   rK   r(   rj   )rn   r   r   r;      s    z:test_no_undesired_distributed_cache_hit.<locals>.<genexpr>c                 S   s   g | ]}g qS r   r   rL   r   r   r   r,      s     receive-from-scatterr   c                 S   s   g | ]}t |d kqS r&   r/   r(   rr   r   r   r,      s     c                 3   s   | ]}t | d V  qdS ))rk   NrK   rq   )rl   rn   r   r   r;      s    c                 S   s   g | ]}t |d kqS r&   rs   rt   r   r   r   r,      s     )N)r?   rA   importorskipZarangerQ   r   r   closer   r
   r@   r4   sumvaluesr]   )r7   listsr   r1   rG   rH   rescountsr   )rl   rn   rm   r   'test_no_undesired_distributed_cache_hitu   s0    


r}   c                   @   s(   e Zd Zdd Zdd ZeZdd ZdS )CountSerializedc                 C   s   || _ d| _d S )Nr   )rV   count)selfrV   r   r   r   __init__   s    zCountSerialized.__init__c                 C   s   | j t|d| S )NrV   )rV   getattr)r   otherr   r   r   __add__   s    zCountSerialized.__add__c                 C   s   |  j d7  _ t| jffS )Nr'   )r   r~   rV   )r   r   r   r   
__reduce__   s    zCountSerialized.__reduce__N)__name__
__module____qualname__r   r   __radd__r   r   r   r   r   r~      s   r~   c                 C   s   | | | | | S r   r   )rE   rF   cder   r   r   add5   s    r   c                 C   sD  t d}t d}t d}t \}\}}t|d | d}td||gdn\}}	tt}
|
|||dd	d
|
|||d	dd
|
||||d	d
|
|||||d
g}dd |D }t |}W 5 Q R X tt	  td| dd W 5 Q R X W 5 Q R X W 5 Q R X W 5 Q R X ||kst
|jdks t
|jdks0t
|jdks@t
d S )Nr'   r	   r>   r5   r6   r   )scatterr[      )r   r   c                 S   s   g | ]\}}}|||qS r   r   )r(   funcr   r   r   r   r   r,      s   z'test_manual_scatter.<locals>.<listcomp>)r7   r   )r[      )r~   r   r   r   r   r   r
   rA   rB   	TypeErrorr@   r   )r7   rV   rW   zrD   rE   rF   r1   rG   rH   r^   rd   expectedresultsr   r   r   test_manual_scatter   s.    (r   c                    sz  t d}|jtd|jd |jtd|jd} gd |gd  }t \}\}}t|d | d`}td"\}}	t dd	 t	|D  W 5 Q R X t
d
|}
|
|d  |
|d   dkstW 5 Q R X W 5 Q R X t \}\}}t|d | dp}td&\}}	t  fdd	tdD  W 5 Q R X t
d
|}
|
|d  dksLt|
|d  dksbtW 5 Q R X W 5 Q R X d S )Nrg   rM   )Zdtyper>   r5   r6   r   c                 s   s&   | ]\}}t t||||d V  qdS ))optNr   r   )r(   r:   rk   r   r   r   r;      s   z$test_auto_scatter.<locals>.<genexpr>rr   r	   c                 3   s$   | ]}t t d d |V  qd S )Nr>   r   r9   Zdata1r   r   r;      s     r   r   )rA   rv   onesrQ   Zuint8r   r   r   r
   	enumerater4   r@   r?   )loop_in_threadrm   Zdata2Zdata_to_processrD   rE   rF   r1   rG   rH   r|   r   r   r   test_auto_scatter   s&    


4&
r   retry_noc                    s   t dd d}fdd fddt f\}\}}t|d | dB}td	. d
t fddt|D }W 5 Q R X W 5 Q R X W 5 Q R X d S )Nrg   r<   c                    s
     | S r   )rx   )rV   r:   j)rm   r   r   my_sum  s    z#test_nested_scatter.<locals>.my_sumc              	      sB   t  }td& t  fddtD }W 5 Q R X t|S )Nr   c                 3   s&   | ]}t  |d  |V  qd S r   rK   )r(   r   )arrayr:   r   r   r   r;     s    zEtest_nested_scatter.<locals>.outer_function_joblib.<locals>.<genexpr>)r   r   r
   r?   rx   )r   r:   r1   r   )NUM_INNER_TASKSr   )r   r:   r   outer_function_joblib  s    
z2test_nested_scatter.<locals>.outer_function_joblibr5   r6   r   i'  c                 3   s$   | ]}t  |d  |V  qd S r   rK   r9   )my_arrayr   r   r   r;     s
   
 z&test_nested_scatter.<locals>.<genexpr>)rA   rv   r   r   r   r   r
   r?   )r7   r   ZNUM_OUTER_TASKSrD   rE   rF   rH   r   )r   r   r   rm   r   r   test_nested_scatter  s    
	

r   c           	         s
  dd  t  \}\}}t|d | d\}tdH\}}tdd fdd	td
D }|D ]}tt|dks\tq\W 5 Q R X W 5 Q R X t|d | d\}tdH\}}tdd fdd	td
D }|D ]}tt|dkstqW 5 Q R X W 5 Q R X W 5 Q R X d S )Nc                  S   sH   t tdddd tdD } | t tdddd tdD O } | S )Nr	   n_jobsc                 s   s   | ]}t tj V  qd S r   r   osgetpidrL   r   r   r   r;   $  s     zOtest_nested_backend_context_manager.<locals>.get_nested_pids.<locals>.<genexpr>c                 s   s   | ]}t tj V  qd S r   r   rL   r   r   r   r;   %  s     )setr
   r?   )Zpidsr   r   r   get_nested_pids#  s     $z<test_nested_backend_context_manager.<locals>.get_nested_pidsr5   r6   r   r	   r   c                 3   s   | ]}t   V  qd S r   rK   rL   r   r   r   r;   +  s   z6test_nested_backend_context_manager.<locals>.<genexpr>r<   c                 3   s   | ]}t   V  qd S r   rK   rL   r   r   r   r;   5  s   )r   r   r   r
   r?   r/   r   r@   )	r   rD   rE   rF   r1   rG   rH   Z
pid_groupsZ	pid_groupr   r   r   #test_nested_backend_context_manager"  s     *r   c                    s   dd   fddt  \}\}}t|d | d}tdz\}}t >} |dks\t|jd	ksjt|fd
dtdD }W 5 Q R X |D ] \}	}
|	dkst|
d	kstqW 5 Q R X W 5 Q R X W 5 Q R X d S )Nc                 S   s
   | j jjS r   )rN   	__class__r   pr   r   r   _backend_typeA  s    zJtest_nested_backend_context_manager_implicit_n_jobs.<locals>._backend_typec               
      s,   t  }  | | jfW  5 Q R  S Q R X d S r   )r
   r   r   )r   r   r   get_nested_implicit_n_jobsD  s    zWtest_nested_backend_context_manager_implicit_n_jobs.<locals>.get_nested_implicit_n_jobsr5   r6   r   r   c                 3   s   | ]}t   V  qd S r   rK   rL   )r   r   r   r;   N  s   zFtest_nested_backend_context_manager_implicit_n_jobs.<locals>.<genexpr>r	   )r   r   r   r
   r@   r   r?   )r7   rD   rE   rF   r1   rG   rH   r   Zall_nested_n_jobsZbackend_typeZnested_n_jobsr   )r   r   r   3test_nested_backend_context_manager_implicit_n_jobs=  s    r   c              
   C   sD   t t}td W 5 Q R X W 5 Q R X dt|j ks@td S )Nr   zcreate a dask client)rA   rB   r   r   strvaluelowerr@   )r7   infor   r   r   test_errorsW  s    
r   c                 C   s   t  \}\}}t|d | d}td@\}}tdddd tdD }t|d	 d	 d	 tsftW 5 Q R X td@\}}tddd
d tdD }t|d	 d	 d	 tstW 5 Q R X W 5 Q R X W 5 Q R X d S )Nr5   r6   r   r	   r   c                 s   s   | ]}t td dV  qd S )NZnested_requirer   outerrL   r   r   r   r;   d  s    z.test_correct_nested_backend.<locals>.<genexpr>r'   r   c                 s   s   | ]}t td dV  qdS )Z	sharedmemr   Nr   rL   r   r   r   r;   j  s   )	r   r   r   r
   r?   rO   r   r@   r   )r7   rD   rE   rF   r1   rG   rH   resultr   r   r   test_correct_nested_backend_  s    $r   c                    s"   t ddd fddtdD S )Nr	   threads)r   Zpreferc                 3   s   | ]}t t V  qd S r   )r   middlerL   r   r   r   r;   q  s    zouter.<locals>.<genexpr>r'   r
   r?   r   r   r   r   r   p  s    r   c                 C   s   t d| ddd tdD S )Nr	   )r   requirec                 s   s   | ]}t t V  qd S r   )r   innerrL   r   r   r   r;   w  s    zmiddle.<locals>.<genexpr>r'   r   )r   r   r   r   r   v  s    r   c                   C   s   t  jS r   )r
   rN   r   r   r   r   r   |  s    r   c              
   C   sN   t | ddd6 td" tdddd td	D  W 5 Q R X W 5 Q R X d S )
NFT)r7   	processesZset_as_defaultr   r[   r   c                 s   s   | ]}t t|V  qd S r   )r   idr9   r   r   r   r;     s     z0test_secede_with_no_processes.<locals>.<genexpr>r	   )r   r   r
   r?   r6   r   r   r   test_secede_with_no_processes  s    
r   c                 C   s   ddl m} | jS )Nr   )
get_worker)r   r   r5   )rH   r   r   r   r   _worker_address  s    r   c                 C   s   t  \}\}}t|d | d}td|d d8\}}t dd tdD }||d gd ksftW 5 Q R X td|d d8\}}t dd tdD }||d gd kstW 5 Q R X W 5 Q R X W 5 Q R X d S )	Nr5   r6   r   )workersc                 s   s   | ]}t t|V  qd S r   r   r   r9   r   r   r   r;     s    z-test_dask_backend_keywords.<locals>.<genexpr>r<   c                 s   s   | ]}t t|V  qd S r   r   r9   r   r   r   r;     s    rU   rC   r   r   r   test_dask_backend_keywords  s    
 
r   c              
   C   s~   t d| dh}td t dd tdD  W 5 Q R X t }|jjjrftd t |d k s@t	q@|j
rpt	W 5 Q R X d S )	NF)r   r7   r   c                 s   s   | ]}t t|V  qd S r   r8   r9   r   r   r   r;     s     ztest_cleanup.<locals>.<genexpr>r<   g{Gz?r   )r   r   r
   r?   r   r   Z	schedulerrd   r   r@   Zfutures)r7   r1   startr   r   r   test_cleanup  s    
"
r   cluster_strategyadaptivelate_scalingz2.1.1z1.28.0z?distributed bug - https://github.com/dask/distributed/pull/2841)reasonc              	   C   s   t dddd}t|}| dkr.|jddd n| dkr@|d z0td t d	d
 tdD  W 5 Q R X W 5 |  |  X d S )Nr   Fr	   ro   r   rp   r   )Zminimummaximumr   r   c                 s   s   | ]}t t|V  qd S r   r8   r9   r   r   r   r;     s     z(test_wait_for_workers.<locals>.<genexpr>r<   )r   r   ZadaptZscalerw   r   r
   r?   )r   r   r1   r   r   r   test_wait_for_workers  s    

&r   c               
   C   s   t dddd} t| }ztddd< d}tjt|d	 t d
d tdD  W 5 Q R X W 5 Q R X tddd< d}tjt	|d	 t dd tdD  W 5 Q R X W 5 Q R X W 5 |  |   X d S )Nr   Fr	   r   r   g?)Zwait_for_workers_timeoutz7DaskDistributedBackend has no worker after 0.1 seconds.)matchc                 s   s   | ]}t t|V  qd S r   r8   r9   r   r   r   r;     s     z0test_wait_for_workers_timeout.<locals>.<genexpr>r<   z+DaskDistributedBackend has no active workerc                 s   s   | ]}t t|V  qd S r   r8   r9   r   r   r   r;     s     )
r   r   rw   r   rA   rB   TimeoutErrorr
   r?   RuntimeError)r   r1   msgr   r   r   test_wait_for_workers_timeout  s    ,0r   rR   Zlokymultiprocessingc                    sl   t dd}t|} fdd}||}| }t|dks@t|d j}t|tsXtdt	|kshtd S )Nr	   )ro   c               	      s:   t jdd$} td ddd tdD  W 5 Q R X | S )NTrecordr	   )r   rR   c                 s   s   | ]}t t|V  qd S r   r8   r9   r   r   r   r;     s    zftest_joblib_warning_inside_dask_daemonic_worker.<locals>.func_using_joblib_parallel.<locals>.<genexpr>r<   )warningscatch_warningsr
   r?   r   rR   r   r   func_using_joblib_parallel  s
    zStest_joblib_warning_inside_dask_daemonic_worker.<locals>.func_using_joblib_parallelr'   r   zdistributed.worker.daemon)
r   r   Zsubmitr   r/   r@   messagerO   UserWarningr   )rR   r   r1   r   Zfutr   warningr   r   r   /test_joblib_warning_inside_dask_daemonic_worker  s    


r   )r   )r   r   )B
__future__r   r   r   r   r   rA   r   uuidr   r   r    r
   r   r   rP   r   r   Z_daskr   rv   r   r   r   r   r   Zdistributed.metricsZdistributed.utils_testr   r   r   r"   r4   rJ   rS   rT   rX   markZparametrizere   r}   objectr~   r   r   r   r0   r?   r   r   r   r   r   r   r   r   r   r   r   r   Zskipif__version__r   r   r   r   r   r   r   <module>   sd   


	
9
%
