U
    2d];                    @   s  d Z ddlmZmZ ddlmZ ddlmZmZ ddl	m
Z
 ddlZddlZddlmZ ddlZddlZddlmZ ddlZddlmZmZmZmZmZmZmZmZ dd	lm Z m!Z! dd
l"m#Z#m$Z$m%Z% ddl&m'Z'm(Z( ddl&m)Z) ddl*m+Z+ ddl*m,Z, ddl*m-Z- ddl.m/Z/ ddl.m0Z0 ddl.m1Z1 ddl.m2Z2 ddl.m3Z3 ddl.m4Z4 ddl.m5Z5 ddl.m6Z6 ddl.m7Z7 ddl.m8Z8 ddl.m9Z9 ddl.m:Z: ddl;m<Z< ddl=m>Z> ddl?m@Z@mAZA ddlBmCZC dd lBmDZD dd!lEmFZF dd"lGmHZH dd#lGmIZI dd$lGmJZJ dd%lKmLZL dd&lKmMZM dd'lKmNZN dd(lKmOZO dd)lKmPZP dd*lKmQZQ dd+lKmRZR dd,lSmTZT dd-lUmVZV dd.lWmXZX dd/lYmZZZm[Z[m\Z\ dd0l]m^Z^ dd1l_m`Z` G d2d3 d3ZaG d4d5 d5e@Zbecd6d6gd7d6gd8d8gd9d8ggZdecd8d8d9d9gZed:d; Zfejghd<e9e
e:d=d>gejghd?deid@fdAdgidgeidBfdAdieidCfgdDdE ZjdFdG ZkdHdI ZldJdK ZmejghdLe7e8gdMdN ZnedOdP ZodQdR ZpdSdT ZqdUdV ZrdWdX ZsdYdZ Ztd[d\ Zud]d^ Zvd_d` Zwdadb Zxdcdd Zydedf Zzdgdh Z{didj Z|G dkdl dle'Z}edmdn Z~dodp Zdqdr Zejghdsd6d9gejghdte8e7gdudv Zdwdx Zdydz Zd{d| Zd}d~ Zedd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zejghde7ddd8d=giife8dd8d=gid9dfgdd Zdd Zdd Zdd Zdd Zejghdte8eD ddd=gide7eD ddd=gidgdd Zejghdte8eIddddd=gidde7eIddddd=giddgdd Zdd Zdd Ze dd Zdd Zdd Zdd Zdd Zdd ZG dd de'Zdd Zdd Zdd ZddĄ ZddƄ ZddȄ Zddʄ Zdd̄ Zdd΄ ZddЄ Zdd҄ ZddԄ Zejghdddgejghde7ddd9dddgiife8dd9dddgiddfgddۄ Zdd݄ Zdd߄ Zdd Zdd Zdd Zdd Zdd Zejghdddgdd Zdd Zdd Zejghde7dddgife8de%d8d׃ifgdd Zejghde7dddgife8de%ddifgdd ZejgdejghdLe7e8gejghdeegdd Zejghdddgdd ZdS (   zTest the search module    )IterableSized)StringIO)chainproduct)partialN)GeneratorType)assert_array_equalassert_array_almost_equalassert_allcloseassert_almost_equalignore_warningsMinimalClassifierMinimalRegressorMinimalTransformer)CheckingClassifierMockDataFrame)	bernoulliexponuniform)BaseEstimatorClassifierMixin)is_classifier)make_classification)
make_blobs)make_multilabel_classification)train_test_split)KFold)StratifiedKFold)StratifiedShuffleSplit)LeaveOneGroupOut)LeavePGroupsOut)
GroupKFold)GroupShuffleSplit)GridSearchCV)RandomizedSearchCV)ParameterGrid)ParameterSampler)BaseSearchCV)FitFailedWarning)	LinearSVCSVC)DecisionTreeRegressor)DecisionTreeClassifier)KMeans)KernelDensity)LocalOutlierFactor)KNeighborsClassifierf1_score)recall_score)accuracy_score)make_scorer)roc_auc_score)confusion_matrix)r2_score)euclidean_distances)SimpleImputer)Pipeline)RidgeSGDClassifierLinearRegression)HistGradientBoostingClassifier)OneTimeSplitterc                   @   sb   e Zd ZdZdddZdd Zdd Zd	d
 Zdd ZeZ	eZ
eZdddZdddZdd ZdS )MockClassifierz8Dummy classifier to test the parameter search algorithmsr   c                 C   s
   || _ d S N	foo_param)selfrE    rG   M/tmp/pip-unpacked-wheel-zrfo1fqw/sklearn/model_selection/tests/test_search.py__init__P   s    zMockClassifier.__init__c                 C   s$   t |t |kstt|| _| S rC   )lenAssertionErrornpuniqueclasses_rF   XYrG   rG   rH   fitS   s    zMockClassifier.fitc                 C   s
   |j d S Nr   shape)rF   TrG   rG   rH   predictX   s    zMockClassifier.predictc                 C   s
   || j  S rC   rD   rF   rP   rG   rG   rH   	transform[   s    zMockClassifier.transformc                 C   s
   || j  S rC   rD   rX   rG   rG   rH   inverse_transform^   s    z MockClassifier.inverse_transformNc                 C   s   | j dkrd}nd}|S )N         ?        rD   )rF   rP   rQ   scorerG   rG   rH   r^   e   s    
zMockClassifier.scoreFc                 C   s
   d| j iS NrE   rD   )rF   deeprG   rG   rH   
get_paramsl   s    zMockClassifier.get_paramsc                 K   s   |d | _ | S r_   rD   )rF   paramsrG   rG   rH   
set_paramso   s    
zMockClassifier.set_params)r   )NN)F)__name__
__module____qualname____doc__rI   rR   rW   rY   rZ   predict_probapredict_log_probadecision_functionr^   ra   rc   rG   rG   rG   rH   rB   M   s   


rB   c                   @   s   e Zd ZdZedd ZdS )LinearSVCNoScorez0A LinearSVC classifier that has no score method.c                 C   s   t d S rC   )AttributeErrorrF   rG   rG   rH   r^   w   s    zLinearSVCNoScore.scoreN)rd   re   rf   rg   propertyr^   rG   rG   rG   rH   rk   t   s   rk   r[      c                    s*   t   fddtt D ks&td S )Nc                    s   g | ]} | qS rG   rG   .0igridrG   rH   
<listcomp>   s     z3assert_grid_iter_equals_getitem.<locals>.<listcomp>)listrangerJ   rK   ru   rG   ru   rH   assert_grid_iter_equals_getitem   s    rz   klass
   n_iterz input, error_type, error_messagez1Parameter .* a dict or a list, got: 0 of type intfooz Parameter .* is not a dict \(0\)z}Parameter (grid|distribution) for parameter 'foo' (is not|needs to be) (a list or a numpy array|iterable or a distribution).*c              	   C   s&   t j||d | | W 5 Q R X d S )Nmatch)pytestraises)r{   inputZ
error_typeerror_messagerG   rG   rH   test_validate_parameter_input   s    r   c               	   C   sv  ddddgi} t | }t|ts$tt|ts2tt|dksBtt| ddgdddgd	}t |}t|d
ksvttdD ]>}tdd |D }|tdd t	|d |d D ks~tq~t| t i }t|dkstt
|i gkstt| tt |d  W 5 Q R X t dddgii ddgig}t|dksDtt
|ddiddii ddigksjtt| d S )Nr   r[   rq         Zhamspameggs)r   bar   c                 s   s"   | ]}t tt|  V  qd S rC   )tupler   sorteditemsrs   prG   rG   rH   	<genexpr>   s     z&test_parameter_grid.<locals>.<genexpr>c                 s   s   | ]\}}d |d|fV  qdS )r   r   NrG   )rs   xyrG   rG   rH   r      s    r   Cr|         ?)r&   
isinstancer   rK   r   rJ   rz   ry   setr   rx   r   r   
IndexError)Zparams1Zgrid1Zparams2Zgrid2rt   ZpointsemptyZ	has_emptyrG   rG   rH   test_parameter_grid   s2    
&r   c               	   C   s   t  } t| ddddgiddd}tj}t t_|tt |t_|jj	dksPt
t|jd jdddg |tt |t |t |t d|_tt |tt W 5 Q R X d S )NrE   r[   rq   r   cvverboseZparam_foo_paramZsklearn)rB   r$   sysstdoutr   rR   rP   r   best_estimator_rE   rK   r	   cv_results_datar^   rh   rj   rY   scoringr   r   
ValueError)clfgrid_searchZ
old_stdoutrG   rG   rH   test_grid_search   s    


r   c                  C   s   t dt fg} dt t gi}t| |dd}|tt |jd }t|d tsVt	t|d tsht	t
|d drzt	t
|d drt	|d |jk	st	|d |jk	st	t
|d d drt	t
|d d drt	d S )NZ	regressorrq   r   Zparam_regressorr   r[   Zcoef_)r<   r?   r=   r$   rR   rP   r   r   r   rK   hasattrr   )pipe
param_gridr   Zregressor_resultsrG   rG   rH   test_grid_search_pipeline_steps   s    
r   SearchCVc              	   C   s   t ddd}t dgd dgd  }tddgd}| |d	dd
dgid
dd}d}tjt|d |j||t 	dd W 5 Q R X d}tjt|d$ |j||t 	dt 
dd W 5 Q R X |j||t 	dt 
dd d S )Nd   r|   r      r[   r   r   )Zexpected_fit_paramsrE   rq   r   raiser   error_scorez0Expected fit parameter\(s\) \['eggs'\] not seen.r   )r   z)Fit parameter spam has length 1; expected)r   r   )rL   arangereshapearrayr   r   r   rK   rR   oneszeros)r   rP   r   r   Zsearchererr_msgrG   rG   rH   test_SearchCV_with_fit_params   s     (r   c               	   C   s   t dd} tddd\}}dddg}tdd}t| d|id	d
}||| t|d|id	d
}||| |j|jkszt||||||kstt|d|i}tj	t
dd |dgg W 5 Q R X d S )Nr   random_staterq   r   centers皙?r[   r|   r   accuracyr   z
no scoringr   )r*   r   rk   r$   rR   best_params_rK   r^   r   r   	TypeError)r   rP   r   CsZclf_no_scorer   Zgrid_search_no_scorerG   rG   rH   test_grid_search_no_score   s    


r   c                  C   s   t ddddd\} }tdd}ddgi}t||d d	| |}t||d
d	| |}tt |dd	| |}t||dd	| |}|| |}|| |}	|| |}
|| |}|dk st|	dk st||	kstt|	| t||
 d S )Nr   rq   皙?r   	n_samples	n_classesZflip_yr   r   r   r   r   r   roc_aucr\   )r   r*   r$   rR   rk   r^   rK   r   )rP   r   r   rv   Zsearch_no_scoringZsearch_accuracyZsearch_no_score_method_aucZ
search_aucZscore_no_scoringZscore_accuracyZscore_no_score_aucZ	score_aucrG   rG   rH   test_grid_search_score_method  s.    

   
r   c               
   C   s   t jd} tdddd\}}| ddd}tdd}ddgi}t tdtdd	t	 g}d
}|D ]H}t
|||d}	tjt|d |	|| W 5 Q R X |	j|||d qbt t g}
|
D ]}t
|||d}	|	|| qd S )Nr      rq   )r   r   r   r   r   r   r[   n_splitsz*The 'groups' parameter should not be None.r   r   )groups)rL   randomRandomStater   randintr*   r    r!   r"   r#   r$   r   r   r   rR   r   r   )rngrP   r   r   r   rv   Z	group_cvs	error_msgr   gsZnon_group_cvsrG   rG   rH   test_grid_search_groups/  s(    

r   c                  C   s   t ddd} t dgd dgd  }dddg}ttddd|i}|| | t|jj	|j	 tt
 d	d
dgi}|| | t|drtttddd|i}t|drtttddd|idd}|| | t|drtd S )Nr   r|   r   r   r[   r   r   r   alphar\   g       @rN   F)refit)rL   r   r   r   r$   r*   rR   r	   r   rN   r=   r   rK   )rP   r   r   r   rG   rG   rH   test_classes__propertyN  s    
r   c                  C   sh   t  } t| ddgidd}|tt t|ds4tt| ddgiddd}|tt t|dsdtd S )NrE   r[   r   r   r   r   )r~   r   )rB   r$   rR   rP   r   r   rK   r%   )r   r   random_searchrG   rG   rH   test_trivial_cv_results_attrg  s    r   c                  C   s   t  } d ddgfD ]}t| ddddgiddd}|tt t|d	sZt|d
rZt|ds^tdD ]8}d| d}tjt	|d t
||t W 5 Q R X qbqd}dD ]<}tjt|d$ t| i |ddddtt W 5 Q R X qd S )Nr   	precisionrE   r[   rq   r   Fr   r   r   best_index_r   )rW   rh   ri   rY   rZ   z`refit=False`. z9 is available only after refitting on the best parametersr   zIFor multi-metric scoring, the parameter refit must be set to a scorer key) r   Trecallr   )accprec)r   r   )rB   r$   rR   rP   r   r   rK   r   r   rl   getattrr   )r   r   r   fn_namer   r   rG   rG   rH   test_no_refits  s8    
    r   c               	   C   sX   t dddd\} }t }t|dddgi}tt || d d | W 5 Q R X d S )	N   r   r   r   
n_featuresr   r   r   r\      )r   r*   r$   r   r   r   rR   )X_y_r   r   rG   rG   rH   test_grid_search_error  s
    r   c                  C   sr   t dddd\} }dgdgdgd}td	d
}t||}|| | tdddd}|| | t|j|jj d S )Nr   r   r   r   r\   rbfr   )r   kernelgammaautor   )r   r+   r$   rR   r	   Z
dual_coef_r   )r   r   
param_dictr   r   rG   rG   rH   test_grid_search_one_grid_point  s    

r   c                  C   sB   t  } d }t| dtddidd}|tt |jjdks>td S )NrE   r[   r   r   r   rq   )	rB   r$   ry   rR   rP   r   r   rE   rK   )r   r   rG   rG   rH   /test_grid_search_when_param_grid_includes_range  s
    r   c               	   C   s6  t dddd\} }ddi}tdd}td	}t||}tjt|d
 || | W 5 Q R X dg i}t }td}t||}tjt	|d
 || | W 5 Q R X ddi}tdd}td}t||}tjt|d
 || | W 5 Q R X dt
di}t }t||}tt	 || | W 5 Q R X d S )Nr|   r   r   r   r   r[   r   r   zParameter grid for parameter 'C' needs to be a list or a numpy array, but got 1 (of type int) instead. Single values need to be wrapped in a list with one element.r   zIParameter grid for parameter 'C' need to be a non-empty sequence, got: []z1,2,3zParameter grid for parameter 'C' needs to be a list or a numpy array, but got '1,2,3' (of type str) instead. Single values need to be wrapped in a list with one element.)r   rq   )r   r+   reescaper$   r   r   r   rR   r   rL   r   )rP   r   r   r   r   searchrG   rG   rH   test_grid_search_bad_param_grid  s<    





r   c                  C   s   t dddd\} }t }t|dddgi}|| d d |d d  || dd  }|jj}t| } t }t|dddgi}|| d d 	 |d d  || dd  }|jj}t
||kd	kst||kstd S )
Nr   r   r   r   r   r   r\   r   g?)r   r*   r$   rR   rW   r   r   sp
csr_matrixZtocoorL   meanrK   )r   r   r   r   y_predr   y_pred2C2rG   rG   rH   test_grid_search_sparse  s    
 r   c                  C   sR  t dddd\} }t }t|dddgidd	}|| d d
 |d d
  || d
d  }|jj}t| } t }t|dddgidd	}|| d d
 |d d
  || d
d  }|jj}t	|| ||kst
dd }t|dd}	t|dddgi|	d	}|| d d
 |d d
  || d
d  }
|jj}||ksDt
t	||
 d S )Nr   r   r   r   r   r   r\   f1r   r   c                 S   s   t | | S rC   r2   )Zy_true_Zy_pred_rG   rG   rH   f1_loss  s    z0test_grid_search_sparse_scoring.<locals>.f1_lossF)Zgreater_is_better)r   r*   r$   rR   rW   r   r   r   r   r	   rK   r6   )r   r   r   r   r   r   r   r   r  ZF1LossZy_pred3ZC3rG   rG   rH   test_grid_search_sparse_scoring  s,    

r  c            	   	   C   s   t dddd\} }t| d d | d d j}|d d }tdd}t|dd	d
gi}||| |jdksrtt| dd  | d d j}|dd  }|	|}t
||kdksttt || | W 5 Q R X d S )Nr   r   r   r   r   precomputedr   r   r   r\   )r   rL   dotrV   r+   r$   rR   best_score_rK   rW   r   r   r   r   tolist)	r   r   K_trainy_trainr   r   ZK_testZy_testr   rG   rG   rH   #test_grid_search_precomputed_kernel  s    

r
  c               	   C   sV   t d} t d}tdd}t|dddgi}tt || | W 5 Q R X d S )N)r|      )r|   r  r  r   r   r\   )	rL   r   r   r+   r$   r   r   r   rR   )r  r	  r   r   rG   rG   rH   3test_grid_search_precomputed_kernel_error_nonsquare9  s    


r  c                   @   s*   e Zd ZdZd	ddZdd Zdd ZdS )
BrokenClassifierz*Broken classifier that cannot be fit twiceNc                 C   s
   || _ d S rC   	parameterrF   r  rG   rG   rH   rI   G  s    zBrokenClassifier.__init__c                 C   s   t | drtd| _d S )Nhas_been_fit_T)r   rK   r  rF   rP   r   rG   rG   rH   rR   J  s    zBrokenClassifier.fitc                 C   s   t |jd S rS   rL   r   rU   rX   rG   rG   rH   rW   N  s    zBrokenClassifier.predict)N)rd   re   rf   rg   rI   rR   rW   rG   rG   rG   rH   r  D  s   
r  c                  C   sX   t ddd} t dgd dgd  }tt dddgigddd	}|| | d S )
Nr   r|   r   r   r[   r  r   Tr   r   )rL   r   r   r   r$   r  rR   )rP   r   r   rG   rG   rH   
test_refitR  s       r  c                  C   sf   dd } t dddd\}}ttdddd	d
dgid| d}||| |jdksTtt|drbtdS )z^
    Test refit=callable, which adds flexibility in identifying the
    "best" estimator.
    c                 S   st   t dddd\}}ttdddddd	gid
dd}||| |jdksLt|j D ]}|| ksVtqV| d  S )z
        A dummy function tests `refit=callable` interface.
        Return the index of a model that has the least
        `mean_test_score`.
        r   r   *   r   r   r   {Gz?r   r[   r   Tr  r   mean_test_score)	r   r$   r*   rR   r   rK   r   keysargmin)
cv_resultsrP   r   r   keyrG   rG   rH   refit_callablef  s    z+test_refit_callable.<locals>.refit_callabler   r   r  r   r   r   r  r   r[   r   r  r   r  N)r   r$   r*   rR   r   rK   r   )r  rP   r   r   rG   rG   rH   test_refit_callable`  s    r  c               	   C   sb   dd } t dddd\}}ttdddd	d
gid| d}tjtdd ||| W 5 Q R X dS )zb
    Test implementation catches the errors when 'best_index_' returns an
    invalid result.
    c                 S   s   dS )zT
        A dummy function tests when returned 'best_index_' is not integer.
        NrG   r  rG   rG   rH   refit_callable_invalid_type  s    zEtest_refit_callable_invalid_type.<locals>.refit_callable_invalid_typer   r   r  r   r   r   r   r[   r   r  z&best_index_ returned is not an integerr   N)r   r$   r*   r   r   r   rR   )r   rP   r   r   rG   rG   rH    test_refit_callable_invalid_type  s    
r!  out_bound_value	search_cvc              	      sf    fdd}t dddd\}}|tdddd	d
gid|d}tjtdd ||| W 5 Q R X dS )zg
    Test implementation catches the errors when 'best_index_' returns an
    out of bound result.
    c                    s    S )zV
        A dummy function tests when returned 'best_index_' is out of bounds.
        rG   r  r"  rG   rH   refit_callable_out_bound  s    z?test_refit_callable_out_bound.<locals>.refit_callable_out_boundr   r   r  r   r   r   r   r[   r   r  zbest_index_ index out of ranger   N)r   r*   r   r   r   rR   )r"  r#  r%  rP   r   r   rG   r$  rH   test_refit_callable_out_bound  s    
r&  c                  C   st   dd } t dddd\}}ttdd}ttdd	d
dddgi|| d}||| |jdksbtt|drptdS )zC
    Test refit=callable in multiple metric evaluation setting
    c                 S   s   d| kst | d  S )z
        A dummy function tests `refit=callable` interface.
        Return the index of a model that has the least
        `mean_test_prec`.
        Zmean_test_prec)rK   r  r  rG   rG   rH   r    s    z8test_refit_callable_multi_metric.<locals>.refit_callabler   r   r  r   r   )ZAccuracyr   r   r   r  r   r[   r  r   r  N)	r   r6   r5   r$   r*   rR   r   rK   r   )r  rP   r   r   r   rG   rG   rH    test_refit_callable_multi_metric  s    	r'  c                  C   s   t ddddd} t dddd}d	d
 }dd }t||dgd}t|ddddgi}|| |tt t	|dst
d S )N,  r|   r   r   rq   i        c                 S   s   | j dd  dkS )Nr[   )r   r   rq   rT   r   rG   rG   rH   check_X  s    z#test_gridsearch_nd.<locals>.check_Xc                 S   s   | j dd  dkS )Nr[   )r)  r*  rT   r+  rG   rG   rH   check_y  s    z#test_gridsearch_nd.<locals>.check_yrR   )r,  r-  methods_to_checkrE   r[   r   )rL   r   r   r   r$   rR   r^   rP   r   r   rK   )ZX_4dZy_3dr,  r-  r   r   rG   rG   rH   test_gridsearch_nd  s    r/  c                  C   s   t ddd} t dgd dgd  }tdd dgd	}td
d}t|dddd
gi|d}||  |	| | t
|dstd S )Nr   r|   r   r   r[   c                 S   s
   t | tS rC   r   rx   r+  rG   rG   rH   <lambda>      z test_X_as_list.<locals>.<lambda>rR   )r,  r.  r   r   rE   rq   r   r   rL   r   r   r   r   r   r$   rR   r  r^   r   rK   rP   r   r   r   r   rG   rG   rH   test_X_as_list  s    
r5  c                  C   s   t ddd} t dgd dgd  }tdd dgd	}td
d}t|dddd
gi|d}|| | 	| | t
|dstd S )Nr   r|   r   r   r[   c                 S   s
   t | tS rC   r0  r+  rG   rG   rH   r1    r2  z test_y_as_list.<locals>.<lambda>rR   )r-  r.  r   r   rE   rq   r   r   r3  r4  rG   rG   rH   test_y_as_list   s    
r6  c                     s   t t fg} z"ddlm}m} | ||f W n tk
r@   Y nX tddd}t	dgd dgd  }| D ]~\  || }} fdd}fd	d
}t
||d}	t|	ddddgi}
|
|||| |
| t|
dsrtqrd S )Nr   )Series	DataFramer   r|   r   r[   c                    s
   t |  S rC   r   r+  )InputFeatureTyperG   rH   check_df!  s    z#test_pandas_input.<locals>.check_dfc                    s
   t |  S rC   r9  r+  )
TargetTyperG   rH   check_series$  s    z'test_pandas_input.<locals>.check_series)r,  r-  rE   rq   r   r   )r   Zpandasr7  r8  appendImportErrorrL   r   r   r   r   r$   rR   r^   rW   r   rK   )typesr7  r8  rP   r   ZX_dfZy_serr;  r=  r   r   rG   )r:  r<  rH   test_pandas_input  s"    

rA  c                  C   s   t ddd\} }tdddd}ddg}d	D ]>}t|td
ddgd||d}|| | |jd dks*tq*t|td
ddgddd}|| | |jd dkstt|td
ddgdd}||  |jd dkstd S )N2   r   r   r   r   r[   )r   initZn_initadjusted_rand_scorefowlkes_mallows_score)rE  rF  rq   r   r   )
n_clusters)r   r   r   rG  r   r   r   )r   r.   r$   dictrR   r   rK   )rP   r   kmr   r   r   rG   rG   rH   test_unsupervised_grid_search/  s,         
rL  c                  C   sv   dd } t ddddgddgddggd\}}tt tdddgd| d	}|| |jd
 dksdt|jdksrtd S )Nc                 S   s   | j dkrdS dS )Nr   r  r   	bandwidth)	estimatorrP   rG   rG   rH   custom_scoringN  s    z2test_gridsearch_no_predict.<locals>.custom_scoringr   r[   r   )cluster_stdr   r   r  rM  rH  rN  r  )r   r$   r/   rJ  rR   r   rK   r  )rP  rP   _r   rG   rG   rH   test_gridsearch_no_predictK  s    $
rS  c                  C   s  ddgt ddd} t| ddd}dd	 |D }t|dks@t|D ]2}|d
 dksXtd|d   krpdksDn tqDddddddddddddgi} t| ddd}dd	 |D dd	 |D kstdt ddi} t| ddd}dd	 |D dd	 |D kstd S )Nr   linearr   r[   )r   r   r|   )param_distributionsr~   r   c                 S   s   g | ]}|qS rG   rG   rs   r   rG   rG   rH   rw   b  s     z&test_param_sampler.<locals>.<listcomp>r   )r   rT  r   rq   r   r   r   r   r)     	   c                 S   s   g | ]}|qS rG   rG   rV  rG   rG   rH   rw   m  s     c                 S   s   g | ]}|qS rG   rG   rV  rG   rG   rH   rw   s  s     )r   r'   rJ   rK   )rU  samplersamplessamplerG   rG   rH   test_param_sampler\  s2            r\  c                    s   | j  t fdd|D s tt fdd|D s:tt fdd|D rTtt fdd|D snt| jr~| j ndg}|D ]} d|  jtj	kstqd S )Nc                 3   s    | ]}t  | tjjV  qd S rC   r   rL   maZMaskedArray)rs   paramr  rG   rH   r   y  s     z/check_cv_results_array_types.<locals>.<genexpr>c                 3   s   | ]} | j tkV  qd S rC   )dtypeobjectrs   r  r  rG   rH   r   z  s     c                 3   s    | ]}t  | tjjV  qd S rC   r]  rb  r  rG   rH   r   {  s     c                 3   s(   | ] }| d s | jtjkV  qdS )ZrankN)
startswithr`  rL   float64rb  r  rG   rH   r   |  s   
r^   zrank_test_%s)
r   allrK   anymultimetric_scorer_r  r`  rL   Zint32)r   
param_keys
score_keysZscorer_keysr  rG   r  rH   check_cv_results_array_typesv  s    rk  c                    sB   t t  t|| d  t fdd|| D s>td S )N)rb   c                 3   s   | ]} | j fkV  qd S rC   rT   rb  r  n_candrG   rH   r     s     z(check_cv_results_keys.<locals>.<genexpr>)r	   r   r  re  rK   )r  ri  rj  rm  rG   rl  rH   check_cv_results_keys  s
    
 rn  c            	         s&  t dddd\} }d}d}tdgdd	gd
dgdtdgddgdg}d}d}|}tt ||dd}|| | |j t d dkst fdd|D st fdd|D stt||| t	 ||| |j t
|jd }t fddt|D stt fddt|D s"td S )NrB  r   r  r   r   r   r   r[   r|   r   r   r   r   polyrq   r   Zdegreeparam_Cparam_degreeparam_gammaparam_kernelr  mean_train_scorerank_test_scoreZsplit0_test_scoreZsplit1_test_scoreZsplit2_test_scoreZsplit0_train_scoreZsplit1_train_scoreZsplit2_train_scorestd_test_scorestd_train_scoremean_fit_timestd_fit_timemean_score_timestd_score_timeT)r   r   return_train_scorery  c                 3   s&   | ]}|d krt  | dkV  qdS )ry  r   Nre  rs   kr  rG   rH   r     s      z.test_grid_search_cv_results.<locals>.<genexpr>c                 3   s.   | ]&}d |kr|dkrt  | dkV  qdS )timery  r[   Nr  r  r  rG   rH   r     s    rb   c                 3   sJ   | ]B} d  | dkr d j | o@ d j | o@ d j |  V  qdS rv  rT  rs  ru  rt  Nmaskrr   r  rG   rH   r     s   c                 3   sL   | ]D} d  | dkr d j |  oB d j |  oB d j | V  qdS rv  r   rs  ru  rt  Nr  rr   r  rG   rH   r     s   )r   rJ  r$   r+   rR   r   re  rK   rk  rn  rJ   ry   )	rP   r   r   Zn_grid_pointsrb   ri  rj  n_candidatesr   rG   r  rH   test_grid_search_cv_results  sR       

	r  c            
         s   t dddd\} }d}d}dgtdd	td
d	ddgddgdg}d}d}|}tt |||dd}|| | |j t||| t ||| t|jd }	t	 fddt
|	D stt	 fddt
|	D std S )NrB  r   r  r   r      r   r|   )Zscaler   ro  rp  rq   rq  rr  rw  T)r~   r   rU  r  rb   c                 3   sJ   | ]B} d  | dkr d j | o@ d j | o@ d j |  V  qdS r  r  rr   r  rG   rH   r     s   z0test_random_search_cv_results.<locals>.<genexpr>c                 3   sL   | ]D} d  | dkr d j |  oB d j |  oB d j | V  qdS r  r  rr   r  rG   rH   r     s   )r   r   r%   r+   rR   r   rk  rn  rJ   re  ry   rK   )
rP   r   r   n_search_iterrb   ri  rj  rm  r   r  rG   r  rH   test_random_search_cv_results  s6    	r  zSearchCV, specialized_paramsr   r   )rU  r~   c                    s  t ddgddgddgddggddddd\}}tj|jd td}d|t|dkd d d d < d|t|dkd d d d < || g| |gg}t |d	d
}| f ||  || t fddt	 j
D } jd d } jd d }	t fddt	 j
D }
 jd d } jd d } jd d dksJtt|ddg t|
ddg |tt|kst|	tt|kst|tdkst|tdkstd S )Nr   r[   r   FP   )r   r   rQ  shuffler   r`  rq   T)rO  r   r  c                    s   g | ]} j d |  d qS )split%d_test_scorer   r   rs   sr   rG   rH   rw   ?  s   z+test_search_default_iid.<locals>.<listcomp>r  rz  c                    s   g | ]} j d |  d qS )zsplit%d_train_scorer   r  r  r  rG   rH   rw   H  s   rx  r{  rs  gUUUUUU?)r   rL   r   rU   boolwherer+   rR   r   ry   	n_splits_r   rK   r   r   approxr   Zstd)r   specialized_paramsrP   r   r  r   Zcommon_paramsZtest_cv_scoresZ	test_meanZtest_stdZtrain_cv_scoresZ
train_meanZ	train_stdrG   r  rH   test_search_default_iid  sF    

	  

r  c                  C   s   t dddd\} }d}tdgddgd	dgd
tdgddgdg}g }ttttdddfD ].}tt |||dd}|| | || q\t	|  d S )NrB  r   r  r   r   r   r[   r|   r   ro  rp  rq   rq  r   r   r   r   F)r   r   r   r   )
r   rJ  r6   r5   r4   r$   r+   rR   r>  *compare_cv_results_multimetric_with_single)rP   r   r   rb   Zgrid_searchesr   r   rG   rG   rH   'test_grid_search_cv_results_multimetric^  s:        r  c                  C   s   t dddd\} }d}d}ttdddtjd	d
dddd}dD ]}g }dD ]\}|rnd}t|trhdn|}nd}t|dd}	t|	|||||d
d}
|
| | |	|
 qNt
|  t|d
 |d | qBd S )NrB  r   r  r   r   r  r[   r   r   )base)r   r   TF)r  r   r   Tr   F)probabilityr   )r~   r   rU  r   r   r   )r   rJ  rL   Zlogspacer   r   r+   r%   rR   r>  r  )compare_refit_methods_when_refit_with_acc)rP   r   r   r  rb   r   Zrandom_searchesr   r  r   r   rG   rG   rH   )test_random_search_cv_results_multimetric  s:    $	  r  c                    s   | j s
ttt| jd | j dd |j D }|dd |j D  t fdddD shtt	j
dd   D d	d | D  d
S )zCompare multi-metric cv_results with the ensemble of multiple
    single metric cv_results from single metric grid/random searchr  c                 S   s    i | ]\}}t d d||qS )_score$Z	_accuracyr   subrs   r  vrG   rG   rH   
<dictcomp>  s     z>compare_cv_results_multimetric_with_single.<locals>.<dictcomp>c                 S   s    i | ]\}}t d d||qS )r  Z_recallr  r  rG   rG   rH   r    s      c                 3   s    | ]}t  | d kV  qdS )r[   N)rL   re  r  Zcv_results_multirG   rH   r     s   z=compare_cv_results_multimetric_with_single.<locals>.<genexpr>)r~  r  r|  r}  c                 S   s    i | ]\}}| d s||qS _timeendswithr  rG   rG   rH   r    s     
  c                 S   s    i | ]\}}| d s||qS r  r  r  rG   rG   rH   r    s     
  N)rg  rK   r	   r   rh  r   r   updatere  rL   testingassert_equal)search_multi
search_accZ
search_recZcv_results_acc_recrG   r  rH   r    s$    

r  c                 C   s   |j |kst|r"| j dks0tn| j r,tdS tdddd\}}dD ]"}tt| ||t||| qFt| ||||| dD ]}t| |t||kstqdS )	zDCompare refit multi-metric search methods with single metric methodsr   Nr   r   r  r   )rW   rh   ri   )r   r  r   )r   rK   r   r   r   r^   )r  r  r   rP   r   methodr  rG   rG   rH   r    s    
 r  	max_depthr   )rO  rU  )rO  r   c              	   C   sJ   t dddd\}}| || d}tjt|d | | W 5 Q R X d S )Nr   r   r  r   z@'DecisionTreeClassifier' object has no attribute 'score_samples'r   )r   rR   r   r   rl   score_samples)r#  rP   r   r   rG   rG   rH   "test_search_cv_score_samples_error  s
    r  T)Znoveltyn_neighborsr   )rO  rU  r   )rO  r   r   c                 C   s   t jd}d}d}t|| }|| }t|dddgddggdddd }t j||jdd	|dfd
gdd}t dg| }d|| d < | || t	| 
|| j
| d S )Nr  r(  g333333?rq   r   r   )r   r   r   rQ  r   ir   )lowhighsize)Zaxisr[   ro   )rL   r   r   intr   Zconcatenater   r   rR   r   r  r   )r#  r   r   Zoutliers_fractionZ
n_outliersZ	n_inliersrP   Zy_truerG   rG   rH   #test_search_cv_score_samples_method  s,    " 
r  c                  C   s   t ddd\} }ddddgi}tt |dd	}tt d
|dd}||fD ]}|| | |j}t|d d |d d  t|d d |d d  t|d d |d d rt	t|d d |d d rt	t|jd ddd
g qHd S )NrB  r  rC  r   r[   gjt?MbP?T)r   r  r   )r~   rU  r  r  r   rx  rq   ry  )
r   r$   r+   r%   rR   r   r   rL   ZallcloserK   )rP   r   r   r   r   r   r  rG   rG   rH   (test_search_cv_results_rank_tie_breaking&  s:       
 

 

 

 
r  c                  C   s~   dgdgdgdgdggdddddg } }t  t f}ddd gi}t }|D ].}t|||d| |}t|jd	 dd g qJd S )
Nr[   rq   r   r   r   r   r   r   Zparam_random_state)r,   r-   r   r$   rR   r	   r   )rP   r   
estimatorsest_parametersr   estr   rG   rG   rH   !test_search_cv_results_none_paramH  s    ( r  c                  C   s2  t dd} dgdgdgdgg}ddddg}t| dddgiddd}t| dddgidddd	}||fD ]}||| d
D ]4}t|j| dkstt|j| dk sxtqxdD ]H}|j| d dkst|j| d dkstt|j| dk stqt|dstt	|j
tst|j
dksdtqdd S )Nr   r   r[   rq   r   r   r   r   )r   r   r~   )r|  r}  )r~  r  r]   refit_time_)r*   r$   r%   rR   rL   re  r   rK   r   r   r  float)ZsvcrP   r   r   rsr   r  rG   rG   rH   test_search_cv_timingW  s2    
r  c               	      sj  d} t dd}tddd\}}dddg}d	D ]6}t|d
|i|| d||j}t| }dtdd t| D  }t	t
||stt| d}	j} t|D ]\ }
|j|
d t
 fddt| D }t|	||D ]x\}\}}||| ||  |dkr,t|| ||| }n&|dkrR||| }t|| |}t|||  qqq,d S )Nr   r   r   rq   r   r   r[   r|   )r   r   r   )r   r   )r  ry  c                 s   s   | ]}d | V  qdS )r  NrG   )rs   Zcv_irG   rG   rH   r     s    z9test_grid_search_correct_score_results.<locals>.<genexpr>r   )r   c                    s   g | ]}j d |    qS r  r  r  Zcandidate_ir   rG   rH   rw     s   z:test_grid_search_correct_score_results.<locals>.<listcomp>r   r   )r*   r   r$   rR   r   rx   r  r   ry   re  rL   Zin1drK   r   r  	enumeraterc   r   splitr3   rW   rj   r7   r   )r   r   rP   r   r   r^   r  Zresult_keysZexpected_keysr   r   Z	cv_scoresrt   traintestcorrect_scoredecrG   r  rH   &test_grid_search_correct_score_results  s:    







r  c                  C   s   t  } t| ddddgiddd}|tt tt|}t|	t|	t t
| ddddgidddd}|tt tt|}t|	t|	t d S )NrE   r[   rq   r   Tr   )r   r~   r   )rB   r$   rR   rP   r   pickleloadsdumpsr
   rW   r%   )r   r   Zgrid_search_pickledr   Zrandom_search_pickledrG   rG   rH   test_pickle  s$         r  c                  C   s  t ddd\} }dddddgi}t }tdd	tdd	g}|D ]}t|||d
}|| | |jd }tt|D ]r}|j	f ||  t
|| |D ]L\}	\}
}|| |
 ||
  || | || }t||jd|	  |  qqrq>|D ]}t|||dd}|| | |jd }tt|D ]v}|j	f ||  t
|| |D ]N\}	\}
}|| |
 ||
  || | || }t||jd|	  |  qFq"qd S )NTr   )Zreturn_indicatorr   r  r[   rq   r   r   r   r   rb   r  )r   r~   )r   r   r,   r-   r$   rR   r   ry   rJ   rc   r  r  r^   r   r%   )rP   r   r  r   r  r  r   Z
res_paramscand_irt   r  r  r  r   rG   rG   rH   &test_grid_search_with_multioutput_data  s@    


r  c                  C   sR   t ddd} dddddg}tdd}t|i dd	| |}t|d
rNtd S )Nr  r   ro   r   r[   F)r  rq   r   rh   )rL   r   r   r+   r$   rR   r   rK   rP   r   r   r   rG   rG   rH   test_predict_proba_disabled  s
    
r  c                  C   s|   t jdt jddd} t j| dd d f< dddddg}tdtd	t jd
fdt fg}t|ddddgidd	| | d S )Nr  r  r   ro   rq   r   r[   Zimputerr   )ZstrategyZmissing_values
classifierZclassifier__foo_paramr   r   )
rL   r   rd  r   nanr<   r;   rB   r$   rR   )rP   r   r   rG   rG   rH   test_grid_search_allows_nans  s    r  c                   @   s:   e Zd ZdZdZdddZdddZdd	 Zdd
dZdS )FailingClassifierz,Classifier that raises a ValueError on fit()rq   Nc                 C   s
   || _ d S rC   r  r  rG   rG   rH   rI     s    zFailingClassifier.__init__c                 C   s   | j tjkrtdd S )Nz%Failing classifier failed as required)r  r  FAILING_PARAMETERr   r  rG   rG   rH   rR     s    zFailingClassifier.fitc                 C   s   t |jd S rS   r  rX   rG   rG   rH   rW     s    zFailingClassifier.predictc                 C   s   dS )Nr]   rG   rO   rG   rG   rH   r^     s    zFailingClassifier.score)N)N)NN)	rd   re   rf   rg   r  rI   rR   rW   r^   rG   rG   rG   rH   r    s   

r  c               	      s  t dddd\} }t }t|ddddgigdd	d
dtjdtjd}tjt|d 	| | W 5 Q R X t
jd }fdd t fddt|D stt|ddddgigdd	tddtjdtjd}tjt|d 	| | W 5 Q R X t
jd }t fddt|D s.tjd }|d dkrT|d dksXt||j dksltj|jks~td S )Nr  r|   r   r   r  r[   rq   r   Fr]   r   r   r   z5 fits failed.+total of 15.+The score on these train-test partitions for these parameters will be set to 0\.0.+5 fits failed with the following error.+ValueError.+Failing classifier failed as requiredflagsr   rb   c                    s    t  fddtjD S )Nc                    s   g | ]} j d |   qS r  r  r  )r   rt   rG   rH   rw   3  s     zPtest_grid_search_failing_classifier.<locals>.get_cand_scores.<locals>.<listcomp>)rL   r   ry   r  rt   )r   r  rH   get_cand_scores1  s    z<test_grid_search_failing_classifier.<locals>.get_cand_scoresc                 3   s4   | ],}j d  | tjkrt |dkV  qdS )param_parameterr]   N)r   r  r  rL   re  rs   r  r  r   rG   rH   r   7  s
   z6test_grid_search_failing_classifier.<locals>.<genexpr>r  z5 fits failed.+total of 15.+The score on these train-test partitions for these parameters will be set to nan.+5 fits failed with the following error.+ValueError.+Failing classifier failed as requiredc                 3   s6   | ].}j d  | tjkrtt |V  qdS )r  N)r   r  r  rL   re  isnanr  r  rG   rH   r   P  s
   ry  r   )r   r  r$   r   compileDOTALLr   warnsr)   rR   rJ   r   re  ry   rK   r  r  r   )rP   r   r   warning_messager  ZranksrG   r  rH   #test_grid_search_failing_classifier  sV    	
 r  c               	   C   sn   t dddd\} }t }t|dtjgd igdd}tjd	tjd
}tjt	|d |
| | W 5 Q R X d S )Nr  r|   r   r   r  r   r]   r   zrAll the 15 fits failed.+15 fits failed with the following error.+ValueError.+Failing classifier failed as requiredr  r   r   r  r$   r  r   r  r  r   r   r   rR   )rP   r   r   r   r  rG   rG   rH   )test_grid_search_classifier_all_fits_fail`  s    r  c               	   C   s\   t dddd\} }t }t|ddddgigdd	d
d}tt || | W 5 Q R X d S )Nr  r|   r   r   r  r[   rq   r   Fr   r  )r   r  r$   r   r   r   rR   r  rG   rG   rH   )test_grid_search_failing_classifier_raiseu  s    	r  c            	   	   C   sZ  ddgdddgdddd	gig} t | d
d}d
}d}d|||f }tjt|d t| W 5 Q R X t | dd}t|}t|dkstt| D ]}||kstqtt | dddksttdtdtdd} t | ddd}t|}t|dkstdd |D }tt	|dkstt
ddddgd}t |dd}t|}t|dksVtd S )Nr   r[   abc)firstsecondthirdZtwovaluesrX  r}   rW  z}The total space of parameters %d is smaller than n_iter=%d. Running %d iterations. For exhaustive searches, use GridSearchCV.r   i  r|   )r  r  r  c   r  )r~   r   c                 S   s&   g | ]}d |d |d |d f qS )z	a%db%dc%dr  r  r  rG   r   rG   rG   rH   rw     s     z7test_parameters_sampler_replacement.<locals>.<listcomp>r   r)  )r'   r   r  UserWarningrx   rJ   rK   r&   ry   r   r   )	rb   rY  r~   Z	grid_sizeZexpected_warningrZ  r  Zhashable_samplesZparams_distributionrG   rG   rH   #test_parameters_sampler_replacement  s8    
r  c                  C   s   ddgi} t ddd}ddddddg}ttdd	| d
d}t|drNt||| || |	| ddgi} ttdd	| d
d}t|drt||| t|drtd S )NlossZlog_loss   r   ro   r   r[   Zhinge)r  r   )rO  r   r   rh   )
rL   r   r   r$   r>   r   rK   rR   rh   ri   )r   rP   r   r   rG   rG   rH   #test_stochastic_gradient_loss_param  s2       

   r  c                  C   sR   t ddd} ddddddg}tdd}t|dddgid	d
}|| | d S )Nr   ro   r   r[   r   r   r   r   r   )r   r   )rL   r   r   r*   r$   rR   r  rG   rG   rH   %test_search_train_scores_set_to_false  s
    
r  c            	         s  d} d}t | dd\}}ttdddddd	git|| d
dd  || ttdddddd	git|ddd}||| tt|ddd||tst	ttdddddd	git|ddd||dd}||| ttdddddd	git|ddddd}||| dd }t
j||j||j t
jdd  j D dd |j D  ttdddddddgit|dddd  || dD ]\i }tdD ]$ fddtdD |< qt|d |d  t|d |d  qd S )Nr   r   r   rC  r   r   r   r   g333333?)r   r   T)r   r   r  r   )r   r  r   c                 S   s   dD ]}|  | q| S )N)r|  r}  r~  r  )pop)r  r  rG   rG   rH   _pop_time_keys  s    z>test_grid_search_cv_splits_consistency.<locals>._pop_time_keysc                 S   s    i | ]\}}| d s||qS r  r  r  rG   rG   rH   r    s     
  z:test_grid_search_cv_splits_consistency.<locals>.<dictcomp>c                 S   s    i | ]\}}| d s||qS r  r  r  rG   rG   rH   r    s     
  )r   r  )r  r  r   c                    s"   g | ]} j d |f   qS )zsplit%d_%s_scorer  r  r   Zparam_iZ
score_typerG   rH   rw   1  s   z:test_grid_search_cv_splits_consistency.<locals>.<listcomp>r[   rq   r   )r   r$   r*   rA   rR   r   r   r  r   rK   rL   r  r  r   r   ry   r
   )	r   r   rP   r   Zgs2Zgs3Zgs4r  Zper_param_scoresrG   r  rH   &test_grid_search_cv_splits_consistency  sv    
 

r  c                  C   sJ   t  } t| ddddgiddd}|tt ||t}tt| d S )NrE   r[   rq   r   r   )rB   r$   rR   rP   r   rZ   rY   r	   )r   r   ZX_round_triprG   rG   rH   +test_transform_inverse_transform_round_trip:  s
    r  c                     s   dd  fddG fdddt } tddtd	d
dd\ | dd }dddgidddgig}|j}|| t|D ]H}|d  r|dd  dkr|dkrt||t||kstd| qd S )Nc                 S   s   |j }t|  t|kst| D ]f}|ds"t| | | |< | | jjdkrnt	|| | | d| d q"t
|| | | d| d q"d S )Nr  Oz	Checking )r   )r   r   r  rK   r  rL   Z
asanyarrayr`  kindr	   r   )resultsgscvZexp_resultsr  rG   rG   rH   check_resultsD  s    
  z-test_custom_run_search.<locals>.check_resultsc                    s   t | dd S )NTr  )r$   rR   rI  )rP   r   r   rG   rH   fit_gridR  s    z(test_custom_run_search.<locals>.fit_gridc                       s*   e Zd Z fddZfddZ  ZS )z.test_custom_run_search.<locals>.CustomSearchCVc                    s   t  j|f| d S rC   superrI   rF   rO  kwargs	__class__rG   rH   rI   V  s    z7test_custom_run_search.<locals>.CustomSearchCV.__init__c                    sd   |ddiddig} |dddgi |ddiddig} |dddgidddgig d S )Nr  r[   rq   min_samples_splitr   r|   rG   )rF   evaluater  r	  r  rG   rH   _run_searchY  s    z:test_custom_run_search.<locals>.CustomSearchCV._run_search)rd   re   rf   rI   r  __classcell__rG   r  r  rH   CustomSearchCVU  s   r  r   r   r   r   )r   Zn_informativer   Tr
  r  r[   rq   r  r   r|   ro   rR  >   r  r   r   rN   zAttribute %s not equal)	r(   r,   r   rR   r   dirislowerr   rK   )r  Zmycvr  r  attrrG   )rP   r	  r   r  r   rH   test_custom_run_searchC  s*    


r  c               	   C   sb   G dd dt } | t tt G dd dt }tjtdd |t tt W 5 Q R X d S )Nc                       s&   e Zd Z fddZdddZ  ZS )z;test__custom_fit_no_run_search.<locals>.NoRunSearchSearchCVc                    s   t  j|f| d S rC   r  r  r  rG   rH   rI   x  s    zDtest__custom_fit_no_run_search.<locals>.NoRunSearchSearchCV.__init__Nc                 [   s   | S rC   rG   )rF   rP   r   r   
fit_paramsrG   rG   rH   rR   {  s    z?test__custom_fit_no_run_search.<locals>.NoRunSearchSearchCV.fit)NN)rd   re   rf   rI   rR   r  rG   rG   r  rH   NoRunSearchSearchCVw  s   r  c                       s   e Zd Z fddZ  ZS )z3test__custom_fit_no_run_search.<locals>.BadSearchCVc                    s   t  j|f| d S rC   r  r  r  rG   rH   rI     s    z<test__custom_fit_no_run_search.<locals>.BadSearchCV.__init__)rd   re   rf   rI   r  rG   rG   r  rH   BadSearchCV  s   r  z_run_search not implemented.r   )r(   r+   rR   rP   r   r   r   NotImplementedError)r  r  rG   rG   rH   test__custom_fit_no_run_searchv  s
    r   c               	   C   sx   t ddt} dd | D  d}tt dddd	gi| d
d}tjtdd" |td | t	d |  W 5 Q R X d S )Nr   r   c                 S   s   g | ]}|qS rG   rG   )rs   urG   rG   rH   rw     s     z0test_empty_cv_iterator_error.<locals>.<listcomp>r   r   r  r  r   r   r   Zn_jobszNNo fits were performed. Was the CV iterator empty\? Were there no candidates\?r   )
r   r  rP   r%   r=   r   r   r   rR   r   )r   
train_sizeridgerG   rG   rH   test_empty_cv_iterator_error  s    r%  c               	   C   st   G dd dt } | dd}d}tt dddd	gi|d
d}tjtdd" |td | td |  W 5 Q R X d S )Nc                   @   s   e Zd Zdd ZdS )z.test_random_search_bad_cv.<locals>.BrokenKFoldc                 _   s   dS )Nr[   rG   )rF   argskwrG   rG   rH   get_n_splits  s    z;test_random_search_bad_cv.<locals>.BrokenKFold.get_n_splitsN)rd   re   rf   r(  rG   rG   rG   rH   BrokenKFold  s   r)  r   r   r   r   r  r  r   r   r"  zXcv.split and cv.get_n_splits returned inconsistent results. Expected \d+ splits, got \d+r   )	r   r%   r=   r   r   r   rR   rP   r   )r)  r   r#  r$  rG   rG   rH   test_random_search_bad_cv  s    
r*  r  Fr   rW  r   c              	   C   s  t ddd\}}G dd d}| t f| d|d|}tt}||| W 5 Q R X |rhdd	gndg}t|t|kstt||D ]"\}	}
d
|
 dt	|	j
kstq|jd  }t|jd }t|jd | | t|jd |  |k std S )Nrq   r   )r   r   c                   @   s    e Zd ZdZdd Zdd ZdS )zHtest_searchcv_raise_warning_with_non_finite_score.<locals>.FailingScorerz1Scorer that will fail for some split but not all.c                 S   s
   d| _ d S rS   )n_countsrm   rG   rG   rH   rI     s    zQtest_searchcv_raise_warning_with_non_finite_score.<locals>.FailingScorer.__init__c                 S   s&   |  j d7  _ | j d dkr"tjS dS )Nr[   r   r   )r+  rL   r  )rF   rO  rP   r   rG   rG   rH   __call__  s    zQtest_searchcv_raise_warning_with_non_finite_score.<locals>.FailingScorer.__call__N)rd   re   rf   rg   rI   r,  rG   rG   rG   rH   FailingScorer  s   r-  r   )r   r   r  r  r  zOne or more of the z scores are non-finitery  r  )r   r-   r   r  r  rR   rJ   rK   zipstrmessager   maxrL   r  r	   re  )r   r  r  rP   r   r-  rv   Zwarn_msgZset_with_warningmsgZdatasetZ	last_rankZnon_finite_maskrG   rG   rH   1test_searchcv_raise_warning_with_non_finite_score  s*    r3  c            	      C   s   dd } t dddd\}}tdd}t|dd	d
gi| dd}||| ddddg}|D ]}d||jksXtqX||}t||}|	||t
|d kstd S )Nc                 S   s2   |  |}t||}|d |d |d |d dS )N)r   r   r   r[   )r[   r   )r[   r[   )tnfpfntp)rW   r8   )r   rP   r   r   cmrG   rG   rH   custom_scorer  s    

zAtest_callable_multimetric_confusion_matrix.<locals>.custom_scorer(   r   r  r   r   r   r   r[   r6  r  r5  r7  r8  zmean_test_{}r4  )r   r*   r$   rR   formatr   rK   rW   r8   r^   r   r  )	r:  rP   r   r  r   Zscore_namesnamer   r9  rG   rG   rH   *test_callable_multimetric_confusion_matrix  s    


r>  c                  C   s   dd } t dddd\}}tdd}t|dd	d
gi| dd}t|dd	d
giddgdd}||| ||| |jt|jkst|j|jkst|	||t|	||kstd S )Nc                 S   s    |  |}t||t||dS )N)r   r   )rW   r4   r5   r  rP   r   r   rG   rG   rH   r:    s    
zHtest_callable_multimetric_same_as_list_of_strings.<locals>.custom_scorerr;  r   r  r   r   r   r   r[   r   r  r   
r   r*   r$   rR   r  r   r  rK   r   r^   )r:  rP   r   r  search_callable
search_strrG   rG   rH   1test_callable_multimetric_same_as_list_of_strings  s(    
 
   
  rC  c                  C   s*  dd } t dddd\}}tdd}t|dd	d
gi| dd}t|dd	d
giddd}t|dd	d
gidgdd}||| ||| ||| |jt|jkst|j|jkst|	||t|	||kst|jt|jkst|j|jkst|	||t|	||ks&td S )Nc                 S   s   |  |}t||S rC   )rW   r4   r?  rG   rG   rH   r:  '  s    
zHtest_callable_single_metric_same_as_single_string.<locals>.custom_scorerr;  r   r  r   r   r   r   r[   Tr  r   r@  )r:  rP   r   r  rA  rB  Zsearch_list_strrG   rG   rH   1test_callable_single_metric_same_as_single_string%  s2    
 
   
  "rD  c               	   C   sf   dd } t dddd\}}ttdddd	d
gi| dd}d}tjt|d ||| W 5 Q R X d S )Nc                 S   s   ddiS )NZbad_namer[   rG   r  rP   r   rG   rG   rH   
bad_scorerC  s    zBtest_callable_multimetric_error_on_invalid_key.<locals>.bad_scorerr;  r   r  r   r   r   r   r[   Z	good_namer  z`For multi-metric scoring, the parameter refit must be set to a scorer key or a callable to refitr   )r   r$   r*   r   r   r   rR   )rF  rP   r   r   r2  rG   rG   rH   .test_callable_multimetric_error_on_invalid_keyA  s    
rG  c               	   C   s   dd } t dddd\}}t }t|dddd	gig| d
dd}tjdtjd}tjt|d |	|| W 5 Q R X t
|jd dddg d S )Nc                 S   s   ddiS Nr   r[   rG   rE  rG   rG   rH   r:  Y  s    zBtest_callable_multimetric_error_failing_clf.<locals>.custom_scorerr  r|   r   r   r  r[   rq   Fr   r  zm5 fits failed.+total of 15.+The score on these train-test partitions for these parameters will be set to 0\.1r  r   Zmean_test_acc)r   r  r$   r   r  r  r   r  r)   rR   r   r   )r:  rP   r   r   r   r  rG   rG   rH   +test_callable_multimetric_error_failing_clfV  s"    rI  c               	   C   s   dd } t dddd\}}t }t|dtjgd ig| d	d
d}d}tjd| tjd}tjt	|d |
|| W 5 Q R X d S )Nc                 S   s   ddiS rH  rG   rE  rG   rG   rH   r:  t  s    zBtest_callable_multimetric_clf_all_fits_fail.<locals>.custom_scorerr  r|   r   r   r  r   Fr   r  z1ValueError: Failing classifier failed as requiredz5All the 15 fits failed.+your model is misconfigured.+r  r   r  )r:  rP   r   r   r   Zindividual_fit_error_messager   rG   rG   rH   +test_callable_multimetric_clf_all_fits_failr  s"    rJ  c                  C   s   d} t | d\}}t }dddgi}t||}t||dd}t|drJtt|drXt||| ||| |j| ks~t|j| kstd S )Nr   )r   Zmax_iterr   r[   r}   n_features_in_)r   r@   r$   r%   r   rK   rR   rK  )r   rP   r   Zgbdtr   r   r  rG   rG   rH   test_n_features_in  s    
rL  pairwisec                    sJ   G  fdddt }| }d}t|ddgi} | d ksFt|dS )z
    Test implementation of BaseSearchCV has the pairwise tag
    which matches the pairwise tag of its estimator.
    This test make sure pairwise tag is delegated to the base estimator.

    Non-regression test for issue #13920.
    c                       s   e Zd Z fddZdS )zStest_search_cv_pairwise_property_delegated_to_base_estimator.<locals>.TestEstimatorc                    s   d iS NrM  rG   rm   rM  rG   rH   
_more_tags  s    z^test_search_cv_pairwise_property_delegated_to_base_estimator.<locals>.TestEstimator._more_tagsN)rd   re   rf   rP  rG   rO  rG   rH   TestEstimator  s   rQ  z.BaseSearchCV pairwise tag must match estimatorr  r|   rM  N)r   r$   	_get_tagsrK   )rM  rQ  r  attr_messager   rG   rO  rH   <test_search_cv_pairwise_property_delegated_to_base_estimator  s
    
rT  c                  C   s\   G dd dt } |  }d}dD ]8}|j|d t|ddgi}|| d kst|qd	S )
z
    Test implementation of BaseSearchCV has the pairwise property
    which matches the pairwise tag of its estimator.
    This test make sure pairwise tag is delegated to the base estimator.

    Non-regression test for issue #13920.
    c                   @   s   e Zd ZdddZdd ZdS )zXtest_search_cv__pairwise_property_delegated_to_base_estimator.<locals>.EstimatorPairwiseTc                 S   s
   || _ d S rC   rO  )rF   rM  rG   rG   rH   rI     s    zatest_search_cv__pairwise_property_delegated_to_base_estimator.<locals>.EstimatorPairwise.__init__c                 S   s
   d| j iS rN  rO  rm   rG   rG   rH   rP    s    zctest_search_cv__pairwise_property_delegated_to_base_estimator.<locals>.EstimatorPairwise._more_tagsN)T)rd   re   rf   rI   rP  rG   rG   rG   rH   EstimatorPairwise  s   
rU  z4BaseSearchCV _pairwise property must match estimatorr  rO  r  r|   rM  N)r   rc   r$   rR  rK   )rU  r  rS  Z_pairwise_settingr   rG   rG   rH   =test_search_cv__pairwise_property_delegated_to_base_estimator  s    	rV  c                  C   s   d} d}t | dd\}}ddgi}t }t|||d}||| ||}t|}tdd	}t|||d}||| ||}	d
}
||	k st|
dS )z
    Test implementation of BaseSearchCV has the pairwise tag
    which matches the pairwise tag of its estimator.
    This test ensures the equivalence of 'precomputed'.

    Non-regression test for issue #13920.
    rB  rq   r   rC  r  r|   r   r  )Zmetricz2GridSearchCV not identical with precomputed metricN)r   r1   r$   rR   rW   r:   re  rK   )r   r   rP   r   Zgrid_paramsr   r   Zpreds_originalZX_precomputedZpreds_precomputedrS  rG   rG   rH   ;test_search_cv_pairwise_property_equivalence_of_precomputed  s    



rW  zSearchCV, param_searchr  r   r  c                 C   sP   G dd dt t}| | |}tdd\}}|j||dd |jjdksLtd S )Nc                   @   s(   e Zd ZdddZd	ddZdd ZdS )
z,test_scalar_fit_param.<locals>.TestEstimatorNc                 S   s
   || _ d S rC   )r  )rF   r  rG   rG   rH   rI     s    z5test_scalar_fit_param.<locals>.TestEstimator.__init__c                 S   s
   || _ d S rC   )r_)rF   rP   r   rrG   rG   rH   rR     s    z0test_scalar_fit_param.<locals>.TestEstimator.fitc                 S   s   t jt|dS )NrT   )rL   r   rJ   rX   rG   rG   rH   rW     s    z4test_scalar_fit_param.<locals>.TestEstimator.predict)N)N)rd   re   rf   rI   rR   rW   rG   rG   rG   rH   rQ    s   

rQ  r  r   )rY  )r   r   r   rR   r   rX  rK   )r   param_searchrQ  modelrP   r   rG   rG   rH   test_scalar_fit_param  s
    
r\  r   c           
      C   sd   t tddddi\}}}}G dd dt}dd }| | |}||f|dd}	|j||f|	 d S )	Nr  r   r   c                       s   e Zd Zd fdd	Z  ZS )z9test_scalar_fit_param_compat.<locals>._FitParamClassifierNc                    s`   t  j|||d |dkstt|s*tt|ts8t|d jdksJt|d jdks\t| S )N)sample_weightr   rq   r[   )r  rR   rK   callabler   r   ndim)rF   rP   r   r]  tuple_of_arraysscalar_paramcallable_paramr  rG   rH   rR   	  s    	z=test_scalar_fit_param_compat.<locals>._FitParamClassifier.fit)NNNN)rd   re   rf   rR   r  rG   rG   r  rH   _FitParamClassifier	  s
       rc  c                   S   s   d S rC   rG   rG   rG   rG   rH   _fit_param_callable(	  s    z9test_scalar_fit_param_compat.<locals>._fit_param_callable)r`  rb  ra  )r   r   r>   rR   )
r   rZ  ZX_trainZX_validr	  Zy_validrc  rd  r[  r  rG   rG   rH   test_scalar_fit_param_compat	  s    re  z)ignore:The total space of parameters 4 is	Predictorc           	      C   s   t jd}|ddt dgd dgd   }}tdt fd| fg}dd	gdd	gd
}| ||dd}||| |j	 |	 kst
||}t|rt|d |||tt||kst
n.t||  |||tt||kst
d S )Nr      rq   r   r[   r  ZtransformerZ	predictorr|   )Ztransformer__paramZpredictor__paramar   r  )rL   r   r   Zrandnr   r<   r   rR   r   r  rK   rW   r   r	   r^   r   r  r5   r   r   r9   )	r   rf  r   rP   r   r[  rb   r   r   rG   rG   rH   1test_search_cv_using_minimal_compatible_estimator<	  s"    (

"rh  c                 C   s   t ddddd\}}tdd}ddgi}t||d	d
d
|d|| |  j}|r`td|}ntd|}t|d
ks|t	dS )zkCheck that search cv with verbose>2 shows the score for single
    metrics. non-regression test for #19658.r   rq   r   r   r   r   r   r   r   r   )r   r   r   r  z$score=\(train=[\d\.]+, test=[\d.]+\)zscore=[\d\.]+N)
r   r*   r$   rR   Z
readouterroutr   findallrJ   rK   )Zcapsysr  rP   r   r   rv   Zcapturedr   rG   rG   rH   test_search_cv_verbose_3[	  s&    

 
rk  )rg   collections.abcr   r   ior   	itertoolsr   r   	functoolsr   r  r   r@  r   r   ZnumpyrL   Zscipy.sparsesparser   r   Zsklearn.utils._testingr	   r
   r   r   r   r   r   r   Zsklearn.utils._mockingr   r   Zscipy.statsr   r   r   Zsklearn.baser   r   r   Zsklearn.datasetsr   r   r   Zsklearn.model_selectionr   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   Zsklearn.model_selection._searchr(   Z#sklearn.model_selection._validationr)   Zsklearn.svmr*   r+   Zsklearn.treer,   r-   Zsklearn.clusterr.   Zsklearn.neighborsr/   r0   r1   Zsklearn.metricsr3   r4   r5   r6   r7   r8   r9   Zsklearn.metrics.pairwiser:   Zsklearn.imputer;   Zsklearn.pipeliner<   Zsklearn.linear_modelr=   r>   r?   Zsklearn.ensembler@   Z$sklearn.model_selection.tests.commonrA   rB   rk   r   rP   r   rz   markZparametrizer   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r
  r  r  r  r  r!  r&  r'  r/  r5  r6  rA  rL  rS  r\  rk  rn  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r%  r*  r3  r>  rC  rD  rG  rI  rJ  rL  rT  rV  rW  r\  re  filterwarningsrh  rk  rG   rG   rG   rH   <module>   s  (
'"%

'
	($
-
P?
8#$&
 




!"
(%.	Q( 	_	3
-
 

3
