U .d/@s dZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddlm Z ddl m Z ddlmZddl Z ddlZddlZddlZddlmZdd lmZmZdd lmZdd lmZmZdd lmZdd lmZmZmZmZm Z edk rddl!m"Z"ddl#m$Z$z ddl%Z%Wne&k r8dZ%YnXzddl'm(Z(Wne&k rddZ(YnXz ddl)Z)Wne&k rdZ)YnXddl*m+Z+ddl*m,Z,ddl*m-Z-ddl*m.Z.ddl*m/Z/ddl*m0Z0ddl1m2Z2m3Z3ddl1m4Z4m5Z5ddl1m6Z6m7Z7ddl1mZm8Z8m9Z9m:Z:ddl;me8?Z@e@dde8D7Z@edkrjgZAnd d!gZAeAd"gZBeCed#re@DeEd$e8e9ZFd%d&ZGd'd(ZHd)d*ZIGd+d,d,eJZKd=d.d/ZLd0d1ZMd>d2d3ZNd4d5ZOd6d7ZPd8d9ZQd:d;ZRejSjde6d>d?fd@gdAdBdCgdDedEdFZTedGe@edHddId>dJgedKdIdLdMgdNdOZUedGe@dPdQZVdRdSZWeedTdUdVdWdXdYdZgd[d\ZXeedGd!d d"gd]d^ZYd_d`ZZedae8edbe8dcddZ[dedfZ\edgdhZ]didjZ^edHddIdkgdldmZ_edGeBdndoZ`edpdqZaedGeBdrdsZbeedGeBdtduZceedGeAdvdwZddxdyZeedGe8edzdd{d|d}d~ddddddddg fdd{d}ddd|d~ddddddg fgddZfeedGeBddZgddZheedGeAddZiddZjddZkddZleedGeBejSjmddddZnGddde+ZoddZpedGe@ddZqddZrddZseedk ddddZtddZueBddZvevwddexdkDeedGevddZyGddde+ZzddZ{ddZ|ddZ}ddZ~Gdddejj/Ze4deeedGd"d!d dgddZeedHdId>dgedGeBddZddZeedeBedeBdd„ZeddĄZddƄZddȄZeddd>dgdd̄Zeddddddddddddg ddلZeddۄZeeedGeAdd݄ZdZeedGeAddZdZdZdZdZeedGeAedkr\gnd$gedeeegeddddgddZdjejejejdZeddZdjejejejdZeddZddZddZeeedGeAddZddZeeddZddZedGe@edHddIdJd>gddZeeedGeAddZedGedkrdd"gndd!d"gddZeedGeAddZeddZd d Zd d Zd dZddZd?ddZeedGd!d"gddZeee)dkddddZd@ddZedGedk rnd"gnd!d"gddZddZedGedk rdd!gndgee(dkdddd Zee)dk d!dd"d#Zd$d%Zd&d'Zd(d)Zd*d+Zd,d-ZeeedHdIddJd>gd.d/Zeeed0ddIddgedHdId>gd1d2ZeedHdId>ged3d4d5d6gd7d8ZeeedGd d"e-e,gd9d:ZeedHdIdd>gd;d<ZdS(Az Test the parallel module. N)format_exceptionsqrt)sleep) PicklingError TimeoutErrorparallel)dumpload)mp)np with_numpy)with_multiprocessing) parametrizeraisescheck_subprocess_callskipifwarns)get_reusable_executor)Queue) parallel_sum)SequentialBackend)ThreadingBackend)MultiprocessingBackend)ParallelBackendBase) LokyBackend) SafeFunction)Paralleldelayed)register_parallel_backendparallel_backend)effective_n_jobs cpu_count)r BACKENDSDEFAULT_BACKENDEXTERNAL_BACKENDS)JoblibException)WorkerInterruptcCsg|]}t|qS)r&).0Z backend_strr+r+=/tmp/pip-unpacked-wheel-ho8k9mhl/joblib/test/test_parallel.py Jsr.multiprocessingloky threading get_contextspawncCst|dt|ddS)N_poolZ_workers)getattrbackendr+r+r- get_workersXsr8cCs||SNr+)xyr+r+r-division\sr<cCs|dSNr+r:r+r+r-square`sr@c@seZdZdZddZdS)MyExceptionWithFinickyInitz1An exception class with non trivial __init__ cCsdSr9r+)selfabcdr+r+r-__init__gsz#MyExceptionWithFinickyInit.__init__N__name__ __module__ __qualname____doc__rGr+r+r+r-rAdsrAFcCs"|dkr|rtddddnt|S)NrCrDrErF)rA ValueError)r:custom_exceptionr+r+r-exception_raiserks rPcCstdtdS)Ng?)timerKeyboardInterruptr?r+r+r-interrupt_raiserrs rScCs|d||S)zO A module-level function so that it can be spawn with multiprocessing. r>r+)r:r;zr+r+r-fwsrUcCsttdSNr)typer get_active_backendr+r+r+r-_active_backend_type~srYcCst||dddtdDS)Nn_jobsr7css|]}tt|VqdSr9r!r@r,ir+r+r- sz parallel_func..r range) inner_n_jobsr7r+r+r- parallel_funcsrdcCstdkstdSrV)r%AssertionErrorr+r+r+r-test_cpu_countsrfcCstdkstdSrV)r$rer+r+r+r-test_effective_n_jobssrgzbackend_n_jobs, expected_n_jobs)r`r`r[Nrz positive-intz negative-intNone)idsc Cs@td|dtdd|ks tW5QRXtdddksverbose dcCs:ddtdDt|||dddtdDks6tdS)NcSsg|] }t|qSr+r@r,r:r+r+r-r.sz(test_simple_parallel..)r[r7rocss|]}tt|VqdSr9r\rsr+r+r-r_sz'test_simple_parallel..rbr re)r7r[ror+r+r-test_simple_parallelsrvc Cs~|jtdddtjdd6}td|ddd td D}|d d d gksRtW5QRXdd|D}t|d ksztdS)NnameZ!some_new_name_for_the_main_thread)targetrwvalueTrecordr>rZcss|]}tt|VqdSr9r\rsr+r+r-r_sz6test_main_thread_renamed_no_warning..r`rrcSsg|]}dt|jkr|qS)zworker timeout)strmessager,wr+r+r-r.sz7test_main_thread_renamed_no_warning..) setattrr1current_threadwarningscatch_warningsr rbrelen)r7 monkeypatchwarninforesultsr+r+r-#test_main_thread_renamed_no_warnings rc Csvtjdd}tdt||dW5QRXdd|D}|rf|rbt|dko`d|d jd kSd S|rntdSdS) NTrzalways)r7rccSsg|] }|jqSr+)r~rr+r+r-r.sz*_assert_warning_nested..rzbacked parallel loops cannotrF)rr simplefilterrdrargsre)r7rcexpectedrr+r+r-_assert_warning_nesteds  rz%parent_backend,child_backend,expected)r0r/T)r0r0F)r/r/T)r/r0T)r1r/T)r1r0Tcsltd|dfddtdDtd|dfddtdD}|dkr\t|shtn t|shtdS)Nr>rZc3s |]}ttdddVqdS)rFr7rcrNr!rr,_ child_backendr+r-r_s z0test_nested_parallel_warnings..rtc3s |]}ttddVqdS)r>rNrrrrr+r-r_s r1)r rbanyreall)parent_backendrrresr+rr-test_nested_parallel_warningss rcCs>dg}dd}tj||fd}|||ds:tdS)NFc SsTtjdd"}tddddtdDW5QRXtt|t|dk|d<dS) NTrzr>ricss|]}ttdVqdS)皙?Nr!rrr+r+r-r_szPtest_background_thread_parallelism..background_thread..r|r)rrr rbprintr)is_run_parallelrr+r+r-background_threads  z=test_background_thread_parallelism..background_thread)rxrr)r1Threadstartjoinre)r7rrtr+r+r-"test_background_thread_parallelisms rcCs"td|dddtdDdS)Nr>rZcss|]}ttdVqdS){Gz?Nr\rr+r+r-r_ sznested_loop..rar6r+r+r- nested_loop srrrcs&td|dfddtdDdS)Nr>rZc3s|]}ttVqdSr9)r!rrrr+r-r_sz#test_nested_loop..ra)rrr+rr-test_nested_loopsrcCstdSr9)rNr6r+r+r-raise_exceptionsrc CsLtt:tddd"}|ttdttdgW5QRXW5QRXdS)Nr>r0rZ)rrNr r!rrr r+r+r-)test_nested_loop_with_exception_with_lokys    rcs<tddtdddfddtdDs8tdS) z1Input is mutable when using the threading backendrt)maxsizer>r1rZc3s|]}tjdVqdSrN)r!putrqr+r-r_'sz1test_mutate_input_with_threads..N)rr rbfullrer+r+rr-test_mutate_input_with_threads$s  rr`cCs6td}dd|Dt|ddd|Dks2tdS)z.Check the keyword argument processing of pmap. cSsg|]}t|ddqSrr;rUrsr+r+r-r.0sz(test_parallel_kwargs..ricss|]}tt|ddVqdSrrNr!rUrsr+r+r-r_1sz'test_parallel_kwargs..Nru)r[lstr+r+r-test_parallel_kwargs,s rc Cstd}dd|D}td|d^}|j}||dd|DksDt||dd|Dks^ttdk r|t|t|jks|tW5QRXtdk rt|jdkst||d d|Dksttdk rt|jdkstdS) NrcSsg|]}t|ddqSrrrsr+r+r-r.7sz4test_parallel_as_context_manager..r|rZcss|]}tt|ddVqdSrrrsr+r+r-r_@sz3test_parallel_as_context_manager..css|]}tt|ddVqdSrrrsr+r+r-r_Ascss|]}tt|ddVqdSrrrsr+r+r-r_Ms)rbr _backendrer r8)r7rrpZmanaged_backendr+r+r- test_parallel_as_context_manager4s rc sLGdddtttdd&tddfddtd DW5QRXd S) ze Check that pmap captures the errors when it is passed an object that cannot be pickled. c@seZdZddZdS)z1test_parallel_pickling..UnpicklableObjectcSs tddS)NZ123) RuntimeErrorrBr+r+r- __reduce__Xsz.UnpicklableObject.__reduce__N)rIrJrKrr+r+r+r-UnpicklableObjectWsrzthe task to sendmatchr>ric3s|]}ttVqdSr9r!idrrr+r-r_\sz)test_parallel_pickling..rN)objectrrr rbr+r+rr-test_parallel_picklingRsrcCs0ttd|ddddtdDdks,tdS)Nr>rr[r7timeoutcss|]}ttdVqdS)gMbP?Nrrsr+r+r-r_bsz0test_parallel_timeout_success..)rr rbrer6r+r+r-test_parallel_timeout_success_s rc Cs8tt&td|ddddtdDW5QRXdS)Nr>rrcss|]}ttdVqdS)rNrrsr+r+r-r_ksz-test_parallel_timeout_fail..r)rrr rbr6r+r+r-test_parallel_timeout_failfs rc Cstdk rtt&td|dddtddDW5QRXtt td|ddddDW5QRXtd|d&}t|jdk stt|j}tt|ddtddDW5QRXt|jdk stt|j|k std dt d D|d d t d Dkstt|j}tt|d ddDW5QRXt|jdk sZtt|j|k sntddt d D|dd t d DkstW5QRXt|jdkstn,tt tdddddDW5QRXtt$tddddtddDW5QRXtt $tddddd t dDW5QRXz"tdddd tddDWn4t k r}zt |trtW5d}~XYn XtddS)Nr>rZcSsg|]\}}tt||qSr+r!r<r,r:r;r+r+r-r.wsz&test_error_capture..)rr)rrcSsg|]}tt|qSr+r!rSrsr+r+r-r.{scSsg|]\}}tt||qSr+rrr+r+r-r.scSsg|]}t|ddqSrrrsr+r+r-r.srcss|]}tt|ddVqdSrrrsr+r+r-r_sz%test_error_capture..cSsg|]}tt|qSr+rrsr+r+r-r.scSsg|]}t|ddqSrrrsr+r+r-r.scss|]}tt|ddVqdSrrrsr+r+r-r_sricSsg|]}tt|qSr+rrsr+r+r-r.scSsg|]\}}tt||qSr+rrr+r+r-r.sr)r[rocss|]}tt|ddVqdS)T)rONr!rPr]r+r+r-r_srcss |]\}}tt||VqdSr9rrr+r+r-r_sz'The excepted error has not been raised.)r rZeroDivisionErrorr zipr*r8rrerbrRrA Exception isinstancer)rN)r7r Zoriginal_workersexr+r+r-test_error_captureosj                 "rcCs|d|dS)Nz Consumed %s)append)queueitemr+r+r-consumersrzbatch_size, expected_queue Produced 0z Consumed 0z Produced 1z Consumed 1z Produced 2z Consumed 2 Produced 3z Consumed 3z Produced 4z Consumed 4z Produced 5z Consumed 5r|csTtfdd}td||dfdd|D|ks@ttdksPtdS) zC Test that with only one job, Parallel does act as a iterator. c3s&tdD]}d||VqdSNz Produced %irbrr^rr+r-producers z'test_dispatch_one_job..producerrr[ batch_sizer7c3s|]}tt|VqdSr9r!rrsrr+r-r_sz(test_dispatch_one_job.. N)listr rer)r7rZexpected_queuerr+rr-test_dispatch_one_jobs  rcst}|fdd}tddd|dfdd|Dt}|d d ksXt|d d d }|dksvt|d}||ksttdkstd S)zW Check that using pre_dispatch Parallel does indeed dispatch items lazily. c3s&tdD]}d||VqdSrrrrr+r-rs z/test_dispatch_multiprocessing..producerr>rr`)r[r pre_dispatchr7c3s|]}ttdVqdS)rNrrrr+r-r_sz0test_dispatch_multiprocessing..rrNr|z Consumed anyrhrr)r Managerrr reindexr)r7managerrZqueue_contentsZfirst_consumption_indexZproduced_3_indexr+rr-test_dispatch_multiprocessings    rc CsFtdddd.}|ddtdD|jdks8tW5QRXdS) Nr>autor1rcss|]}tt|VqdSr9rr]r+r+r-r_sz/test_batching_auto_threading..rr rbrZcompute_batch_sizere)rr+r+r-test_batching_auto_threadingsrc CsFtdd|d.}|ddtdD|jdks8tW5QRXdS)Nr>rrcss|]}tt|VqdSr9rr]r+r+r-r_sz2test_batching_auto_subprocesses..rrr)r7rr+r+r-test_batching_auto_subprocessessrc Cs8tt&tddddddtdDW5QRXdS) zCMake sure that exception raised during dispatch are indeed capturedr>r)r[rrocss|]}tt|VqdSr9rr]r+r+r-r_sz*test_exception_dispatch..rN)rrNr rbr+r+r+r-test_exception_dispatchs rcCs tddddtdDdS)Nr>ricss|]}tt|VqdSr9rr,jr+r+r-r_sz(nested_function_inner..rrarr+r+r-nested_function_innersrcCs tddddtdDdS)Nr>ricss|]}tt|VqdSr9)r!rrr+r+r-r_sz(nested_function_outer..rrarr+r+r-nested_function_outersrz'https://github.com/joblib/loky/pull/255)reasonc Cstt$}td|dddtdDW5QRXt|j|j|j}d|}d|ksZt d|ksft d |ksrt t|jtkst d S) zEnsure errors for nested joblib cases gets propagated We rely on the Python 3 built-in __cause__ system that already report this kind of information to the user. r>rZcss|]}tt|VqdSr9)r!rr]r+r+r-r_+sz1test_nested_exception_dispatch..rrrrPN) rrNr rbrrWrytbrre)r7excinfoZ report_linesreportr+r+r-test_nested_exception_dispatch!s     rc@s$eZdZdZdddZd ddZdS) FakeParallelBackendz8Pretends to run concurrently while running sequentially.rNcKs|||_||_|Sr9)r$r[r )rBr[r Z backend_argsr+r+r- configure<s zFakeParallelBackend.configurecCs"|dkrttd|d}|S)Nrr)maxr r%rBr[r+r+r-r$Asz$FakeParallelBackend.effective_n_jobs)rN)r)rIrJrKrLrr$r+r+r+r-r9s rc Csntt}tddW5QRXdt|jks0ttt}tdW5QRXW5QRXdt|jksjtdS)Nz unit-testingr6zInvalid backend:)rrNr r}ryrer#)rr+r+r-test_invalid_backendGs   rc Cs:tt}td|dW5QRXdt|jks6tdS)NrrZz&n_jobs == 0 in Parallel has no meaning)rrNr Z_initialize_backendr}ryre)r7rr+r+r-test_invalid_njobsRs rcCs8z*tdtdtksttdtks(tW5td=XdS)NZ test_backend)r&r"rrer+r+r+r-test_register_parallel_backendYs   rcCsNttkstz$tdtdddttks0tW5tt_XttksJtdS)Nr1T)Z make_default)rYDefaultBackendrer'r r"r&rr+r+r+r-test_overwrite_default_backendbs rzOnly without multiprocessingc Csdttdd"tddddtdDW5QRXtdtddtdDW5QRXdS) Nz)joblib backend '.*' is not available on.*rr0r6css|]}tt|VqdSr9r\r]r+r+r-r_rsz2test_backend_no_multiprocessing..r`css|]}tt|VqdSr9r\r]r+r+r-r_vs)r UserWarningr rbr#r+r+r+r-test_backend_no_multiprocessingns & rc Cst|ddt\}}|dks&ttddks6tt}|jdksJt|dkrvt|tksbtt|j tkstn|dkrt|t kstt|j t kstnX|dkrt|t kstt|j t kstn,| drt|t kstt|j t kstW5QRXdS)Nr`rir/r0r1Ztest_)r#r rXrer$r r[rWrrrr startswithr) backend_nameactive_backend active_n_jobsrr+r+r-check_backend_context_managerys$   r cCsg|] }d|qS)ztest_backend_%dr+r]r+r+r-r.scCsf|tkr|t|tttks$tt|ttks:ttdddddtDttksbtdS)Nr>r1rZcss|]}|stt|VqdSr9)r!r )r,rDr+r+r-r_sz/test_backend_context_manager..) r&setitemrrYrrer r all_backends_for_context_manager)rr7r+r+r-test_backend_context_managersrc@seZdZdZdddZdS)ParameterizedParallelBackendz9Pretends to run conncurrently while running sequentially.NcCs|dkrtd||_dS)Nzparam should not be None)rNparam)rBrr+r+r-rGsz%ParameterizedParallelBackend.__init__)NrHr+r+r+r-rsrc Cs|tdtttksttddddtt\}}t |tksHt|j dksVt|dksbtt }|j dksvt|j |kst|ddtdD}W5QRX|dd tdDkstttkstdS) NZ param_backend*r`)rr[css|]}tt|VqdSr9r!rr]r+r+r-r_sz=test_parameterized_backend_context_manager..rtcSsg|] }t|qSr+rr]r+r+r-r.sz>test_parameterized_backend_context_manager..)rr&rrYrrer#r rXrWrr r[rrb)rr r rrr+r+r-*test_parameterized_backend_context_managers   rc Csttkstttddddtt\}}t|tks>t|jdksLt|dksXtt }|j dkslt|j |kszt|ddt dD}W5QRX|ddt dDkstttkstdS) N+)rrtricss|]}tt|VqdSr9rr]r+r+r-r_szFtest_directly_parameterized_backend_context_manager..cSsg|] }t|qSr+rr]r+r+r-r.szGtest_directly_parameterized_backend_context_manager..) rYrrer#rr rXrWrr r[rrb)r r rrr+r+r-3test_directly_parameterized_backend_context_managers   rcCstdtS)Nr)rosgetpidr+r+r+r-sleep_and_return_pidsrcCs<ttksttdks ttddddtdDS)Nrr>ricss|]}ttVqdSr9)r!rrr+r+r-r_sz"get_nested_pids..)rYrrer _effective_n_jobsrbr+r+r+r-get_nested_pidss rcs eZdZdZfddZZS) MyBackendz:Backend to test backward compatibility with older backendscstt|dSrV)superrget_nested_backendr __class__r+r-rszMyBackend.get_nested_backend)rIrJrKrLr __classcell__r+r+r r-rsrZback_compat_backendc CsRt|@tddddtdD}|D]}tt|dks*tq*W5QRXdS)Nr>ricss|]}ttVqdSr9)r!rrr+r+r-r_sz6test_nested_backend_context_manager..rr)r#r rbrsetre)r7Z pid_groupsZ pid_groupr+r+r-#test_nested_backend_context_managers  r$c shddtddfddtdDtd*tddfddtdDW5QRXdS) NcSs0tt|kstt|}t|ks,tdSr9)rYr&rer$r r)Zexpected_backend_typeZexpected_n_jobr+r+r-check_nested_backend sz?test_nested_backend_in_sequential..check_nested_backendrric3s|]}tddVqdS)r0rNr!r)r%r+r-r_sz4test_nested_backend_in_sequential..rc3s|]}tVqdSr9r&rr7r%r[r+r-r_s)r rbr#)r7r[r+r'r-!test_nested_backend_in_sequentials r(c Cs*t|\}}|j|kstW5QRXdSr9)r# nesting_levelre) inner_backendZexpected_levelr7r[r+r+r-check_nesting_level"sr+ outer_backendr*c sdt|dtd|dfddtdDtdd"tfddtdDW5QRXdS) Nrr>rZc3s|]}ttdVqdSrr!r+rr*r+r-r_.sz-test_backend_nesting_level..rric3s|]}ttdVqdSrr-rr.r+r-r_4s)r+r rbr#)r,r*r+r.r-test_backend_nesting_level's r/c sxddlGfdddt}td|ddtd8\}}tddfd d td D|jd ksjtW5QRXdS) NrcseZdZdZjddZdS)z)test_retrieval_context..MyBackendrcss|jd7_dVdSrjrrr+r+r-retrieval_context?sz;test_retrieval_context..MyBackend.retrieval_contextN)rIrJrKr^contextmanagerr0r+) contextlibr+r-r<srZ retrievalcSstddddt|DS)Nr>ricss|]}tt|VqdSr9rr]r+r+r-r_Gsz>test_retrieval_context..nested_call..ra)nr+r+r- nested_callFsz+test_retrieval_context..nested_callr>ric3s|]}t|VqdSr9r&r])r4r+r-r_Jsz)test_retrieval_context..rtr)r2rr"r#r rbr^re)rbarr+)r2r4r-test_retrieval_context8s r6cCstd}t|t|dS)NZfoobar)r)reprpickledumps)er+r+r-test_joblib_exceptionSsr;c CsNtt}tt|ddW5QRXtt}tt|dW5QRXdS)Nrrr:)rr<rrrSr*)Z safe_divisionZsafe_interruptr+r+r-test_safe_function\s   r<rgQ?c Cs"ttt|dW5QRXdS)Nr)rrNr r=r+r+r-test_invalid_batch_sizefs r>z)n_tasks, n_jobs, pre_dispatch, batch_size)r>r>rr)r>r>r[r)rr>r[r)ir>r[r)rr|r[r)rr[r)r 2 * n_jobsr)rrr)rBrrArM)r?rrArcCsJ|||d}ddt|D}tf|ddt|D}||ksFtdS)N)r[rrcSsg|] }t|qSr+rrr]r+r+r-r.~sz0test_dispatch_race_condition..css|]}tt|VqdSr9r\r]r+r+r-r_sz/test_dispatch_race_condition..ru)Zn_tasksr[rrparamsrrr+r+r-test_dispatch_race_conditionls rDcCs8t}tddd}|jd}|}||ks4tdS)Nr>r/rZcontext)r get_start_methodr Z _backend_argsgetre)Zmp_start_methodrrEZ start_methodr+r+r-test_default_mp_contexts   rHcs^|dkrtd}tjd}|ddtjtd|dfddt dDdS) Nr/r3rir>rZc3s |]}ttjjVqdSr9)r!rdotTr]rCr+r-r_szAtest_no_blas_crash_or_freeze_with_subprocesses..) r r2rrandomZ RandomStaterandnrIrJr rb)r7rngr+rKr-.test_no_blas_crash_or_freeze_with_subprocessess   rOafrom joblib import Parallel, delayed def square(x): return x ** 2 backend = "{}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) cCsB|dkrtdkrtdt|}ttjd|gddddS)Nr/forkzVRequire fork start method to use interactively defined functions with multiprocessing.z-cr\[0, 1, 4, 9, 16\]r stdout_regex) r rFpytestskip,UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_NO_MAINformatrsys executable)r7coder+r+r-2test_parallel_with_interactively_defined_functionss   r[aimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed def run(f, x): return f(x) {define_func} if __name__ == "__main__": backend = "{backend}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) callable_position = "{callable_position}" if callable_position == "delayed": print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) elif callable_position == "args": print(Parallel(n_jobs=2, backend=backend)( delayed(run)(square, i) for i in range(5))) else: print(Parallel(n_jobs=2, backend=backend)( delayed(run)(f=square, x=i) for i in range(5))) z!def square(x): return x ** 2 zcdef gen_square(): def square(x): return x ** 2 return square square = gen_square() zsquare = lambda x: x ** 2 define_funccallable_positionr!rkwargsc Csv|dkr$|tkstjdkr$tdtj|||tj tj t j d}| d}| |ttj|jgddddS) N)r/r3win32zNot picklable with pickle)r\r7r]joblib_root_folderzunpicklable_func_script.pyrrQrR) SQUARE_MAINrXplatformrTrU)UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_MAINrWrpathdirnamejoblib__file__rwriterrYstrpath)r7r\r]tmpdirrZZ code_filer+r+r-0test_parallel_with_unpicklable_functions_in_argss&    rkaimport sys import faulthandler # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed from functools import partial class MyClass: '''Class defined in the __main__ namespace''' def __init__(self, value): self.value = value def square(x, ignored=None, ignored2=None): '''Function defined in the __main__ namespace''' return x.value ** 2 square2 = partial(square, ignored2='something') # Here, we do not need the `if __name__ == "__main__":` safeguard when # using the default `loky` backend (even on Windows). # To make debugging easier faulthandler.dump_traceback_later(30, exit=True) # The following baroque function call is meant to check that joblib # introspection rightfully uses cloudpickle instead of the (faster) pickle # module of the standard library when necessary. In particular cloudpickle is # necessary for functions and instances of classes interactively defined in the # __main__ module. print(Parallel(n_jobs=2)( delayed(square2)(MyClass(i), ignored=[dict(a=MyClass(1))]) for i in range(5) )) )r`cCs.|d}|tttj|jgddddS)Nz(joblib_interactively_defined_function.pyrQ)rSr)rrh5INTERACTIVE_DEFINED_FUNCTION_AND_CLASS_SCRIPT_CONTENTrrXrYrirjscriptr+r+r-Btest_parallel_with_interactively_defined_functions_default_backend4s   roauimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed, hash import multiprocessing as mp mp.util.log_to_stderr(5) class MyList(list): '''MyList is interactively defined by MyList.append is a built-in''' def __hash__(self): # XXX: workaround limitation in cloudpickle return hash(self).__hash__() l = MyList() print(Parallel(n_jobs=2)( delayed(l.append)(i) for i in range(3) )) cCs0|d}|tttj|jgdddddS)Nz)joblib_interactive_bound_method_script.pyz\[None, None, None\]Z LokyProcess)rSZ stderr_regexr)rrh9INTERACTIVELY_DEFINED_SUBCLASS_WITH_METHOD_SCRIPT_CONTENTrrXrYrirmr+r+r-5test_parallel_with_interactively_defined_bound_method]s   rrcCs"tg}tdd|gkstdS)Nr>ri)iterr re)Zexhausted_iteratorr+r+r-%test_parallel_with_exhausted_iteratorgsrtcCs"t|tjstdt||S)Nz#Expected np.memmap instance, got %r)rrmemmap TypeErrorrWcopyrKr+r+r- check_memmapls  rxcCsdd}tdd|ddd|dD}t||t|D]\}}tj||q:td d|dd d|dD}t||t|D]\}}tj||qdS) Ncss(t|D]}tjdtjd|VqdS)NrZdtype)rbronesZfloat32)r3r^r+r+r-generate_arrays{s zBtest_auto_memmap_on_arrays_from_generator..generate_arraysr>rr[ max_nbytesr7css|]}tt|VqdSr9r!rxr,rCr+r+r-r_sz.rqr|css|]}tt|VqdSr9r~rr+r+r-r_s)r rrrtestingassert_array_equal)r7r{rresultrr+r+r-)test_auto_memmap_on_arrays_from_generatorssrcCs|Sr9r+)argr+r+r-identitysrcs|dj}tj}tj|ddtj|ddg}t||t|ddt ddfdd d D\}t d tj svt d j |kst tj||dS) Nz test.mmapuint8ryr)Z mmap_moder>ric3s|]}ttVqdSr9)r!rrrur+r-r_sz.test_memmap_with_big_offset..)rr)rrimmapZALLOCATIONGRANULARITYrzerosrzr r r rrureoffsetrr)rjfnamesizeobjrr+rr-test_memmap_with_big_offsets   rc Csrtjdd"}tddddtdDW5QRXt|dksDt|d}t|jts\tt |jd ksntdS) NTrzr)rcss|]}tt|VqdSr9r\r]r+r+r-r_szFtest_warning_about_timeout_not_supported_by_backend..2rzThe backend class 'SequentialBackend' does not support timeout. You have set 'timeout=1' in Parallel but the 'timeout' parameter will not be used.) rrr rbrrerr~rr})rrr+r+r-3test_warning_about_timeout_not_supported_by_backends&rc Csbdgdgd}tt(t}t||ddd|DW5QRXt|}|dks^tdS)NrCrrqrZcss|]}ttj|VqdSr9r!rQrr]r+r+r-r_sz%test_abort_backend..)rrvrQr re)r[r7ZdelaysZt_startdtr+r+r-test_abort_backends  rc Cs|j}tdd||d<}|ddtjdgdDtt|dksNtW5QRXtdD]}t|srqt d q`td tdd|d }|d dtjdgdDtdD]}t|sqt d qtd dS) Nr>r)r[r}r7Z temp_foldercss|]}tt|VqdSr9r~rr+r+r-r_sz(test_memmapping_leaks..rrrqrz/temporary directory of Parallel was not removedr|css|]}tt|VqdSr9r~rr+r+r-r_s) rir rrLrrlistdirrerbr)r7rjrrr+r+r-test_memmapping_leakss&         rcCs<td|dddtdD}|ddtdDks8tdS)Nr>rZcss|]}tdd|VqdS)cSs|dSr=r+r?r+r+r-z2test_lambda_expression...Nr&r]r+r+r-r_sz)test_lambda_expression..rcSsg|] }|dqS)r>r+r]r+r+r-r.sz*test_lambda_expression..)r rbre)r7rr+r+r-test_lambda_expressionsrcsd}d}d|td||d}|fddt|D|jj|jjksLt|jj|jjks`t|fddt|D|jj|jjkst|jj|jjkstd S) zCTest that a parallel backend correctly resets its batch statistics.r>ig@r)ror[r7c3s|]}ttjVqdSr9rr]Z task_timer+r-r_sz6test_backend_batch_statistics_reset..c3s|]}ttjVqdSr9rr]rr+r-r_sN)r rbrZ_effective_batch_sizeZ_DEFAULT_EFFECTIVE_BATCH_SIZEreZ_smoothed_batch_durationZ _DEFAULT_SMOOTHED_BATCH_DURATION)r7r[Zn_inputsrr+rr-#test_backend_batch_statistics_resets$rc CsdD]v}tt|djtks tt|dd}t|jtks>tt|dd}t|jtks\tt|dd}t|jtkstqtdd dd }t|jtksttd dd0tdd }t|jtkst|jdkstW5QRXtd dd6td dd}t|jtks t|jd kstW5QRXtd dd4tdd }t|jtksRt|jdksbtW5QRXtd dd6td dd}t|jtkst|jd kstW5QRXdS)N)rr>rhrithreadsr[prefer processes sharedmemr[requirer>r0)r[r7rrr`rr)rWr rrrerr#r[)r[rr+r+r-$test_backend_hinting_and_constraintss4       rc Cs@Gdddt}t|Btddd}t|j|ks:ttddd}t|j|ksXtW5QRXGdd d t}t|tddd}t|j|kst|\}}|d kst|d ksttddd d }t|jtkst|\}}d }||kst|d kstW5QRXt t t|ddW5QRXdS)Nc@s$eZdZdZdZddZddZdS)z[test_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackendTcSsdSr9r+rr+r+r- apply_async5szgtest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.apply_asynccSs|Sr9r+rr+r+r-r$8szltest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.effective_n_jobsNrIrJrKZsupports_sharedmemZ use_threadsrr$r+r+r+r-MyCustomThreadingBackend1srr>rrrrc@s$eZdZdZdZddZddZdS)z\test_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackendFcSsdSr9r+rr+r+r-rFszhtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.apply_asynccSs|Sr9r+rr+r+r-r$Iszmtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.effective_n_jobsNrr+r+r+r-MyCustomProcessingBackendBsrrr)r[rrozUsing ThreadingBackend as joblib.Parallel backend instead of MyCustomProcessingBackend as the latter does not provide shared memory semantics.r7r) rr#r rWrreZ readouterrrstriprrN)Zcapsysrrrouterrrr+r+r-9test_backend_hinting_and_constraints_with_custom_backends.s*          rc CstttddW5QRXtttddW5QRXtttdddW5QRXtdk rtttdddW5QRXtttd ddW5QRXdS) Ninvalidrrrr)rrr0rr/)rrNr r r+r+r+r-,test_invalid_backend_hinting_and_constraintsbs     rcCsFtj}td}ttjts"t|ttjt|ksBtdS)Nr1)r rr#rrre unregisterrW)defaultZpbr+r+r-test_global_parallel_backendws rc Cs:dd}|td<tdttjts,tW5QRXdS)NcSs ttd<dS)Nfoo)rr&r+r+r+r- register_foosz,test_external_backends..register_foor)r(r#rr rrre)rr+r+r-test_external_backendss rc sxtddd}t|jj|jjfg}dkr:|W5QRS|fddtdD}||dW5QRSQRXdS)zCPerform nested parallel calls and introspect the backend on the wayr>rirc3s(|] }ttfddiVqdS)limitrN)r!_recursive_backend_infor]r^rr+r-r_sz*_recursive_backend_info..rN)r rWrrIr)rb)rr^rZ this_levelrr+rr-rs rc Csrt|dd t}W5QRXtdkr2d}d}nd}d}|d}|df|dfd|fd|fg}||ksntdS)Nr>rirrrZBackendr)r#rr%titlere)r7backend_types_and_levelsZsecond_level_backend_typeZ max_levelZtop_level_backend_typeZexpected_types_and_levelsr+r+r-test_nested_parallelism_limits  rzThis test requires daskc Cstjddd}tjtdtjd}tdD]H}tdt|d}W5QRXt |dks\t t dd |Ds*t q*td t}W5QRXt |dkst t d d |Dst dS) Nr>)Z n_workersZthreads_per_workergcArydask)datar|css|]\}}|dkVqdSZDaskDistributedBackendNr+r,rwrr+r+r-r_sz4test_nested_parallelism_with_dask..css|]\}}|dkVqdSrr+rr+r+r-r_s) distributedZClientrrzintrrbr#rrrer)clientrr^rr+r+r-!test_nested_parallelism_with_dasks   rcCstddtdDS)z6A horrible function that does recursive parallel callscss|]}ttVqdSr9)r!_recursive_parallelr]r+r+r-r_sz&_recursive_parallel..r>ra)Z nesting_limitr+r+r-rsrc Cstt|dd tt }tW5QRXW5QRX|j}|dkrbddlm}t||rpt dnt|t spt dS)Nr>rir0r)TerminatedWorkerErrorz1Loky worker crash when serializing RecursionError) r#r BaseExceptionrryZ&joblib.externals.loky.process_executorrrrTxfailRecursionErrorre)r7rexcrr+r+r-test_thread_bomb_mitigations    rcCs*i}dD]}tj|||<q|tdfS)N)OMP_NUM_THREADSOPENBLAS_NUM_THREADSMKL_NUM_THREADSZVECLIB_MAXIMUM_THREADSZNUMEXPR_NUM_THREADSZNUMBA_NUM_THREADS ENABLE_IPCrq)renvironrGr)Zenv_varsvarr+r+r-_run_parallel_sumsrzNeed OpenMP helper compiledcCstd|dddtdD}ttdd}|D]Z\}}||ksFt|D]<\}}|drr|t|kstqN|dks~t|dksNtqNq2dS) Nr>rZcss|]}ttVqdSr9)r!rrr+r+r-r_sz-test_parallel_thread_limit..rZ_THREADSr1)r rbrr%reitemsendswithr})r7rZexpected_num_threadsZworker_env_varsZomp_num_threadsrwryr+r+r-test_parallel_thread_limits    rz%This test requires dask NOT installedc Cs$ttddtdW5QRXdS)NzPlease install daskrr)rrNr#r+r+r+r-)test_dask_backend_when_dask_not_installedsrc CsdGdddt}d}t|<tjt|d"tddddtdDW5QRXW5QRXdS) Nc@s&eZdZddZdddZddZdS) z3test_zero_worker_backend..ZeroWorkerBackendc_sdSrVr+)rBrr^r+r+r-rsz=test_zero_worker_backend..ZeroWorkerBackend.configureNcSs tddS)NzNo worker availabler)rBfunccallbackr+r+r-r sz?test_zero_worker_backend..ZeroWorkerBackend.apply_asynccSsdSrVr+rr+r+r-r$szDtest_zero_worker_backend..ZeroWorkerBackend.effective_n_jobs)N)rIrJrKrrr$r+r+r+r-ZeroWorkerBackends rz&ZeroWorkerBackend has no active workerrr>ricss|]}tt|VqdSr9rr]r+r+r-r_sz+test_zero_worker_backend..)rr#rTrrr rb)rZ expected_msgr+r+r-test_zero_worker_backends   rcsdadddksttddfddtdD}t|dhksLtdadks^ttddfd dtdD}t|dhkstdS) Nzoriginal valuecSstSr9)MY_GLOBAL_VARIABLEr+r+r+r- check_globalssz@test_globals_update_at_each_parallel_call..check_globalsr>ric3s|]}tVqdSr9r&r]rr+r-r_%sz.z changed valuec3s|]}tVqdSr9r&r]rr+r-r_.s)rrer rbr#)Zworkers_global_variabler+rr-)test_globals_update_at_each_parallel_callsrcCs4ddl}|jdd}|||ddlm}|S)Nrrq)threadpool_info)ZnumpyrLrMrIZ threadpoolctlr)rrCrr+r+r-_check_numpy_threadpool_limits8s   rcCs8|D] }|d|dkr|dSqtd|dS)Nfilepath num_threadsz,An unexpected module was loaded in child: {})rNrW) child_module parent_info parent_moduler+r+r-_parent_max_num_threads_forCs rcCs@|D]6}|D],}t||}t|||h}|d|ks tq qdS)Nr)rminre)Z workers_inforrZchild_threadpool_inforZparent_max_num_threadsrr+r+r-check_child_num_threadsKsrcCsbt}t|dkrtjddt|dddtdD}t|}tt|d}t |||dS) Nr&Need a version of numpy linked to BLASmsgricss|]}ttVqdSr9r!rr]r+r+r-r_csz6test_threadpool_limitation_in_child..r>r) rrrTrUr rbr$rr%r)r[rworkers_threadpool_infosexpected_child_num_threadsr+r+r-#test_threadpool_limitation_in_childWs  rinner_max_num_threadsc Cst}t|dkrtjddtd|d"t|dddtd D}W5QRXt|}|dkrttt |d }n|}t |||dS) Nrrrr0rricss|]}ttVqdSr9rr]r+r+r-r_{sz>test_threadpool_limitation_in_child_context..r>r) rrrTrUr#r rbr$rr%r)r[rrrrr+r+r-+test_threadpool_limitation_in_child_contextms   rvar_namerrrc stddddtj}zdtj<t|dfddtd D}|ddgks`ttd d d (t|dfd dtd D}W5QRX|ddgkstW5|dkrtj=n |tj<XdS)NTZreusecSs tj|Sr9)rrrG)rr+r+r-_get_envsz>test_threadpool_limitation_in_child_override.._get_env4ric3s|]}tVqdSr9r&r]rrr+r-r_sz?test_threadpool_limitation_in_child_override..r>r0rrc3s|]}tVqdSr9r&r]rr+r-r_sr) rshutdownrrrGr rbrer#)r[rZoriginal_var_valuerr+rr-,test_threadpool_limitation_in_child_overrides"    rc Cs(ttddt|ddW5QRXdS)Nz#does not acc.*inner_max_num_threadsrrr)rrer#r6r+r+r-1test_threadpool_limitation_in_child_context_errorsrcCsJdd}||tdd}tdD]"}||tdd}||ks"tq"dS)Ncs,tdt|dfddtdDdS)Nrric3s|]}ttVqdSr9)r!sumr]r?r+r-r_szAtest_loky_reuse_workers..parallel_call..)rbr rir+r?r- parallel_callsz.test_loky_reuse_workers..parallel_callTrr)rrbre)r[rZfirst_executorrexecutorr+r+r-test_loky_reuse_workerss   r)F)rr)r`)N)rLrrXrQrrr1 tracebackrmathrrr8rr/rrTrfr r r Zjoblib._multiprocessing_helpersr Zjoblib.test.commonrrrZjoblib.testingrrrrrZjoblib.externals.lokyrrrposix ImportErrorZ _openmp_test_helper.parallel_sumrrZjoblib._parallel_backendsrrrrrrZjoblib.parallelr r!r"r#r$r%r&r'r(Zjoblib.my_exceptionsr)r*sortedkeysZALL_VALID_BACKENDSZPROCESS_BACKENDSZPARALLEL_BACKENDShasattrrr2rr8r<r@rrArPrSrUrYrdrfrgmarkrmrvrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr rextendrbrrrrrrZ_parallel_backendsrr$r(r+r/r6r;r<r>rDrHrOrVr[rcraZ SQUARE_LOCALZ SQUARE_LAMBDArkrWrdrergrlrorqrrrtrxrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr+r+r+r-s                                         I                         ( ,      #. 4       !