U
    2d!B                    @   sn  d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
Z
ddlZddlmZmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZ ddlmZ ddlmZm Z  ddlm!Z! ddlm"Z" ddlm#Z# ddlm$Z$ ddlm%Z% ddlm&Z& ddlm'Z' ddlm(Z( ddlm)Z) ddlm*Z* ddlm+Z+ ddlm,Z, ddl-m.Z. ddl-m/Z/ ddl-m0Z0 ddl1m2Z2 ddl1m3Z3 dd l1m4Z4 dd!l1m5Z5 dd"l6m7Z7 dd#l6m8Z8 dd$l6m9Z9 dd%l6m:Z: dd&l6m;Z; dd'l6m<Z< dd(l6m=Z= dd)l6m>Z> dd*l6m?Z? dd+l@mAZAmBZBmCZC dd,l@mDZDmEZE dd-lFmGZG dd.lHmIZI dd/lJmKZKmLZL dd0lMmNZN dd1lOmPZP dd2lQmRZR dd3lSmTZT dd4lUmVZV dd5lWmXZX dd6lYmZZZ dd7lYm[Z[ dd8l\m]Z] dd9l^m_Z_ dd:l1m`Z` dd;l1maZa dd<lbmcZc dd=lmdZd zee W n efk
r>   dZeY nX G d>d? d?eZZgG d@dA dAegZhG dBdC dCeZZiG dDdE dEeiZjG dFdG dGZkeldHZmeemZneodddIdIdJdJdKdKdLdLg
ZpeodIdIdIdJdJdJdKdKdKdKg
ZqeerdMZsdNdO ZtdPdQ ZudRdS ZvdTdU ZwdVdW ZxdXdY ZydZd[ Zzd\d] Z{e
j|}d^d_d` Z~dadb Zdcdd Zdedf Zdgdh Zdidj Zdkdl Zdmdn Zdodp Zdqdr Zdsdt Zdudv Zdwdx Zdydz Zd{d| Zd}d~ Zdd Zdd Ze
j|}d^dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd ZG ddĄ deGZddƄ ZddȄ Zddʄ Zdd̄ Zdd΄ ZddЄ Ze
j|}d^dd҄ ZddԄ Zddք ZG dd؄ deZZe
j|dejdgddۄ Ze
j|dejdgdd݄ Zdd߄ Ze
j|}de
j|dejddgdd Ze
j|}de
j|dejddge
j|dddge
j|dddgdd Zdd Ze
j|ddedJdddfdeeddKdddfdeedddddfgdd Zdd Zdd Zdd ZdS )zTest the validation module    N)partial)sleep)
coo_matrix
csr_matrix)FitFailedWarning)FailingClassifier)assert_almost_equal)assert_array_almost_equal)assert_array_equal)assert_allclose)CheckingClassifierMockDataFrame)_num_samples)cross_val_scoreShuffleSplit)cross_val_predict)cross_validate)permutation_test_score)KFold)StratifiedKFold)LeaveOneOut)LeaveOneGroupOut)LeavePGroupsOut)
GroupKFold)GroupShuffleSplit)learning_curve)validation_curve)_check_is_permutation)_fit_and_score)_score)make_regression)load_diabetes)	load_iris)load_digits)explained_variance_score)make_scorer)accuracy_score)confusion_matrix)precision_recall_fscore_support)precision_score)r2_score)mean_squared_error)check_scoring)RidgeLogisticRegressionSGDClassifier)PassiveAggressiveClassifierRidgeClassifier)RandomForestClassifier)KNeighborsClassifier)SVC	LinearSVC)KMeans)MLPRegressor)SimpleImputer)LabelEncoder)Pipeline)StringIO)BaseEstimator)clone)OneVsRestClassifier)shuffle)make_classification)make_multilabel_classification)OneTimeSplitter)GridSearchCVc                   @   s<   e Zd ZdZdd ZdddZdd Zdd	d
Zdd ZdS )MockImprovingEstimatorz+Dummy classifier to test the learning curvec                 C   s   || _ d| _d | _d S Nr   )n_max_train_sizestrain_sizesX_subset)selfrF    rJ   Q/tmp/pip-unpacked-wheel-zrfo1fqw/sklearn/model_selection/tests/test_validation.py__init__Z   s    zMockImprovingEstimator.__init__Nc                 C   s   || _ |jd | _| S rE   rH   shaperG   rI   rH   Zy_subsetrJ   rJ   rK   fit_   s    zMockImprovingEstimator.fitc                 C   s   t d S NNotImplementedErrorrI   XrJ   rJ   rK   predictd   s    zMockImprovingEstimator.predictc                 C   s2   |  |rdt| j| j  S t| j| j S d S )Ng       @)_is_training_datafloatrG   rF   rI   rU   YrJ   rJ   rK   scoreg   s    
zMockImprovingEstimator.scorec                 C   s
   || j kS rQ   rH   rT   rJ   rJ   rK   rW   n   s    z(MockImprovingEstimator._is_training_data)N)NN	__name__
__module____qualname____doc__rL   rP   rV   r[   rW   rJ   rJ   rJ   rK   rD   W   s   

rD   c                       s4   e Zd ZdZd	 fdd	Zdd Zd
ddZ  ZS )!MockIncrementalImprovingEstimatorz*Dummy classifier that provides partial_fitNc                    s   t  | d | _|| _d S rQ   )superrL   xexpected_fit_params)rI   rF   re   	__class__rJ   rK   rL   u   s    z*MockIncrementalImprovingEstimator.__init__c                 C   s
   | j |kS rQ   rd   rT   rJ   rJ   rK   rW   z   s    z3MockIncrementalImprovingEstimator._is_training_datac              	   K   s   |  j |jd 7  _ |d | _| jrt| jt| }|rNtdt| d| D ]F\}}|| jkrVt|t|krVtd| dt| dt| dqVd S )Nr   zExpected fit parameter(s) z
 not seen.zFit parameter z has length z; expected .)	rG   rN   rd   re   setAssertionErrorlistitemsr   )rI   rU   yparamsmissingkeyvaluerJ   rJ   rK   partial_fit}   s"    
z-MockIncrementalImprovingEstimator.partial_fit)N)N)r^   r_   r`   ra   rL   rW   rs   __classcell__rJ   rJ   rf   rK   rb   r   s   rb   c                   @   s<   e Zd ZdZdddZdd Zdd Zdd
dZdd Zd	S )MockEstimatorWithParameterz-Dummy classifier to test the validation curve      ?c                 C   s   d | _ || _d S rQ   )rH   param)rI   rw   rJ   rJ   rK   rL      s    z#MockEstimatorWithParameter.__init__c                 C   s   || _ |jd | _| S rE   rM   rO   rJ   rJ   rK   rP      s    zMockEstimatorWithParameter.fitc                 C   s   t d S rQ   rR   rT   rJ   rJ   rK   rV      s    z"MockEstimatorWithParameter.predictNc                 C   s   |  |r| jS d| j S )N   )rW   rw   )rI   rU   rn   rJ   rJ   rK   r[      s    z MockEstimatorWithParameter.scorec                 C   s
   || j kS rQ   r\   rT   rJ   rJ   rK   rW      s    z,MockEstimatorWithParameter._is_training_data)rv   )NNr]   rJ   rJ   rJ   rK   ru      s   

ru   c                       s(   e Zd ZdZ fddZdd Z  ZS )%MockEstimatorWithSingleFitCallAllowedz<Dummy classifier that disallows repeated calls of fit methodc                    s&   t | drtdd| _t ||S )Nfit_called_zfit is called the second timeT)hasattrrk   rz   rc   rP   rO   rf   rJ   rK   rP      s    z)MockEstimatorWithSingleFitCallAllowed.fitc                 C   s   t d S rQ   rR   rT   rJ   rJ   rK   rV      s    z-MockEstimatorWithSingleFitCallAllowed.predict)r^   r_   r`   ra   rP   rV   rt   rJ   rJ   rf   rK   ry      s   ry   c                	   @   sH   e Zd ZdZdddZdddZd	d
 Zdd ZdddZdddZ	dS )MockClassifierz-Dummy classifier to test the cross-validationr   Fc                 C   s   || _ || _d S rQ   aallow_nd)rI   r~   r   rJ   rJ   rK   rL      s    zMockClassifier.__init__Nc                 C   sT  || _ || _|	| _|
dk	r"|
|  | jr8|t|d}|jdkrP| jsPtd|dk	r|jd |jd kst	d
|jd |jd |dk	r|jd tttkst	d
|jd ttt|dk	rd}|jd |jd kst	|
|jd |jd |dk	rPd	}|jtjksPt	|
|jd |jd
 tjd tjd
 | S )zThe dummy arguments are to test that this fit function can
        accept non-array arguments through cross-validation, such as:
            - int
            - str (this is actually array-like)
            - object
            - function
        N   zX cannot be dr   zKMockClassifier extra fit_param sample_weight.shape[0] is {0}, should be {1}zIMockClassifier extra fit_param class_prior.shape[0] is {0}, should be {1}zRMockClassifier extra fit_param sparse_sample_weight.shape[0] is {0}, should be {1}zUMockClassifier extra fit_param sparse_param.shape is ({0}, {1}), should be ({2}, {3})rx   )	dummy_int	dummy_str	dummy_objr   reshapelenndim
ValueErrorrN   rk   formatnpuniquern   P_sparse)rI   rU   rZ   sample_weightclass_priorsparse_sample_weightsparse_paramr   r   r   callbackfmtrJ   rJ   rK   rP      sP      
 
zMockClassifier.fitc                 C   s&   | j r|t|d}|d d df S )Nr   r   )r   r   r   rI   TrJ   rJ   rK   rV      s    zMockClassifier.predictc                 C   s   |S rQ   rJ   r   rJ   rJ   rK   predict_proba   s    zMockClassifier.predict_probac                 C   s   ddt | j  S )N      ?rx   )r   absr~   rY   rJ   rJ   rK   r[      s    zMockClassifier.scorec                 C   s   | j | jdS )Nr}   r}   )rI   deeprJ   rJ   rK   
get_params  s    zMockClassifier.get_params)r   F)	NNNNNNNNN)NN)F)
r^   r_   r`   ra   rL   rP   rV   r   r[   r   rJ   rJ   rJ   rK   r|      s    
         
?
r|   )
      rx   r   r         c               	   C   s|  t  } tddD ]}|| _t| tt}t|| tt t	ttd d d g}t| t
|}t|| t
| t| t
t}t|| t
t t| t
|}t|| t
| qdd }t|d} t| t t dd}t|d	} t| tt dd}tt t| ttd
d W 5 Q R X td d d d tjf }t dd} t| |t}t dd} tt t| |tdd W 5 Q R X d S )Nir   r   c                 S   s
   t | tS rQ   
isinstancerl   rh   rJ   rJ   rK   <lambda>(      z&test_cross_val_score.<locals>.<lambda>check_Xr   cvcheck_yZsklearnscoringT)r   Fraiseerror_score)r|   ranger~   r   rU   y2r
   r[   r   column_stackX_sparser   tolistpytestraisesr   newaxis)clfr~   scoresmultioutput_y
list_checkX_3drJ   rJ   rK   test_cross_val_score  s2    



r   c                  C   s@   t dd\} }tdd}t|dddgid}t|| |d	d
 d S )NTZ
return_X_yauto)gammaCrx   r   )Z
param_gridr   )n_jobs)r"   r4   rC   r   )rU   rn   r   ZgridrJ   rJ   rK   test_cross_validate_many_jobs<  s    
r   c               	   C   s  t dd\} }t }d}tjt|d" t|| |ttttfd W 5 Q R X tjt|d t|| |ttfd W 5 Q R X tjt|d d t|| |dd W 5 Q R X tjt|d d t|| |d	d W 5 Q R X tjt|d t|| |ttggd W 5 Q R X d
}tjtdd t|| |t	 d W 5 Q R X tjt|d t|| |dd W 5 Q R X tt
}dtj d}tjt|d t|| ||d W 5 Q R X tjt|d t|| |d|id W 5 Q R X tjtdd tt | |dd W 5 Q R X d S )Nr   random_statez.*must be unique strings.*matchr   zEmpty list.*rJ   zDuplicate.*)f1_micror   zB.*scoring is invalid.*Refer to the scoring glossary for details:.*zAn empty dictr   [Scoring failed. The score on this train-test partition for these parameters will be set to z. Details: 
Zfooz#'mse' is not a valid scoring value.Zmse)r@   r|   r   r   r   r   r%   r)   r&   dictr(   r   nanwarnsUserWarningr4   )rU   rn   	estimatorerror_message_regexpZmulticlass_scorerwarning_messagerJ   rJ   rK   )test_cross_validate_invalid_scoring_paramF  sB     "r   c                  C   sf   t dd\} }tdt fdt fg}t|| |dd}|d }t|tsLttdd |D sbtd S )	NTr   imputer
classifier)return_estimatorr   c                 s   s   | ]}t |tV  qd S rQ   )r   r:   ).0r   rJ   rJ   rK   	<genexpr>  s     z7test_cross_validate_nested_estimator.<locals>.<genexpr>)	r"   r:   r8   r|   r   r   rl   rk   all)rU   rn   ZpipelineresultsZ
estimatorsrJ   rJ   rK   $test_cross_validate_nested_estimator  s    r   c               	   C   s  t  } tddd\}}tdd}tddd\}}tddd}|||f|||ffD ]2\}}}	t|	dd}
t|	d	d}g }g }g }g }g }| ||D ]\}}t||| || }	|	|
|	|| ||  |	||	|| ||  |	|
|	|| ||  |	||	|| ||  |	|	 qt
|}t
|}t
|}t
|}t
|}|||||f}t|	||| t|	||| qPd S )
N   r   	n_samplesr   r   linearkernelr   neg_mean_squared_errorr   r2)r   r    r-   r@   r4   r,   splitr=   rP   appendr   array"check_cross_validate_single_metric!check_cross_validate_multi_metric)r   ZX_regZy_regregZX_clfZy_clfr   rU   rn   estZ
mse_scorerZ	r2_scorertrain_mse_scorestest_mse_scorestrain_r2_scorestest_r2_scoresfitted_estimatorstraintestr   rJ   rJ   rK   test_cross_validate  sB    
 




r   c                 C   s<  |\}}}}}dD ]\}	}
|	r@t | ||ddd}t|d | nt | ||ddd}t|ts`tt||
ksptt|d | |	rt | ||dgdd}t|d	 |d nt | ||dgdd}t|tstt||
kstt|d
 | qt | ||ddd}t|d D ].\}}t|j|| j t|j	|| j	 qd S )N))Tr   )Fr   r   Tr   return_train_scoretrain_scoreF
test_scorer   train_r2test_r2)r   r   r   )
r   r	   r   r   rk   r   	enumerater   Zcoef_Z
intercept_)r   rU   rn   r   r   r   r   r   r   r   Zdict_lenZmse_scores_dictZr2_scores_dictkr   rJ   rJ   rK   r     sp                        r   c              	   C   s  |\}}}}}dd }	dt tdd|	f}
ddddh}|d	d
h}dD ]X}|
D ]L}|rt| |||dd}t|d	 | t|d
 | nt| |||dd}t|tstt|	 |r|n|kstt|d | t|d | t
|d tjkstt
|d tjkstt
|d tjks*tt
|d tjksBtt|d dksZtt|d dk srtt|d dkstt|d dk sRtqRqHd S )Nc                 S   s"   |  |}t||t|| dS )Nr   r   )rV   r*   r+   )r   rU   rn   y_predrJ   rJ   rK   custom_scorer  s    

z8check_cross_validate_multi_metric.<locals>.custom_scorerr   r   r   Ztest_neg_mean_squared_errorZfit_timeZ
score_timer   Ztrain_neg_mean_squared_error)TFTr   Fr   r   )r%   r*   unionr   r	   r   r   rk   rj   keystyper   Zndarrayr   )r   rU   rn   r   r   r   r   r   r   r   Zall_scoringZkeys_sans_trainZkeys_with_trainr   r   
cv_resultsrJ   rJ   rK   r     sz    


         

 r   c               
   C   s   t dddd\} }tdd}t tdt t g}d}|D ]X}tjt|d t	|| ||d	 W 5 Q R X tjt|d t
|| ||d	 W 5 Q R X q:d S )
N   r   r   )r   	n_classesr   r   r   z*The 'groups' parameter should not be None.r   )r   rU   rn   r   )r@   r4   r   r   r   r   r   r   r   r   r   )rU   rn   r   Z	group_cvserror_messager   rJ   rJ   rK   #test_cross_val_score_predict_groups@  s    
r  z(ignore: Using or importing the ABCs fromc                     s   t t fg} z"ddlm}m} | ||f W n tk
r@   Y nX | D ]N\  tt }} fdd}fdd}t||d}t	|||dd qFd S )	Nr   Series	DataFramec                    s
   t |  S rQ   r   rh   InputFeatureTyperJ   rK   r   d  r   z-test_cross_val_score_pandas.<locals>.<lambda>c                    s
   t |  S rQ   r  rh   
TargetTyperJ   rK   r   e  r   r   r   r   r   )
r   pandasr  r  r   ImportErrorrU   r   r   r   typesr  r  X_dfy_sercheck_dfcheck_seriesr   rJ   r  r	  rK   test_cross_val_score_pandasV  s    
r  c                  C   s   t dd} t }|j|j }}td}t| |||d}td}g }|||D ]J\}}tjt	|t
d}	tjt	|t
d}
d|	|< d|
|< |||f qNt| |||d}t|| d S )Nr   r   r   r   dtyperx   )r4   r"   datatargetr   r   r   r   zerosr   boolr   r
   )svmirisrU   rn   ZkfoldZscores_indicesZcv_masksr   r   Z
mask_trainZ	mask_testZscores_masksrJ   rJ   rK   test_cross_val_score_maskj  s    
r  c               	   C   s   t dd} t }|j|j }}t||j}t| ||}t dd} t| ||}t|| t dd d} t| ||}t|| t dd} t	
t t| || W 5 Q R X t	
t t| | | W 5 Q R X d S )NZprecomputedr   r   c                 S   s   t | |jS rQ   )r   dotr   )rd   rn   rJ   rJ   rK   r     r   z2test_cross_val_score_precomputed.<locals>.<lambda>)r4   r"   r  r  r   r  r   r   r	   r   r   r   r   )r  r  rU   rn   Zlinear_kernelZscore_precomputedZscore_linearZscore_callablerJ   rJ   rK    test_cross_val_score_precomputed}  s     




r  c               	      s   t  } tjd }ttt}ttdgtdgtdgffdd}tt	d}d dt
  fdd	}t|t|d
| || |d}t| tt|d d S )Nr   rx   )r   rx   rN   r   *   Z42c                    s.   | j  kst| jkst| jks*td S rQ   )r   rk   r   r   )r   Z	DUMMY_INTZ	DUMMY_OBJZ	DUMMY_STRrJ   rK   assert_fit_params  s    z:test_cross_val_score_fit_params.<locals>.assert_fit_paramsr   )r   r   r   r   r   r   r   r   
fit_params)r|   rU   rN   r   r   r   rn   r   r   eyeobjectonesfullr   )r   r   r   ZW_sparser   r#  r%  rJ   r"  rK   test_cross_val_score_fit_params  s,    
" 	
r*  c               	      sl   t  } g   fdd}tjdd  t|}t| tt|dd}W 5 Q R X t|dddg t dksht	d S )Nc                    s     | |f dS )Nr   )r   )Zy_testZ	y_predictZ_score_func_argsrJ   rK   
score_func  s    z3test_cross_val_score_score_func.<locals>.score_funcTrecordr   )r   r   r   )
r|   warningscatch_warningsr%   r   rU   rn   r
   r   rk   )r   r,  r   r[   rJ   r+  rK   test_cross_val_score_score_func  s    r1  c               	   C   s4   G dd d} t t t|  t W 5 Q R X d S )Nc                   @   s   e Zd ZdS )z4test_cross_val_score_errors.<locals>.BrokenEstimatorN)r^   r_   r`   rJ   rJ   rJ   rK   BrokenEstimator  s   r2  )r   r   	TypeErrorr   rU   )r2  rJ   rJ   rK   test_cross_val_score_errors  s    r4  c                  C   s   t  } tdd}t|| j| j}t|dddddgd t|| j| jdd}t|dddddgd t|| j| jdd}t|dddddgd d S )	Nr   r   
ףp=
?r   r   accuracyr   Zf1_weighted)r"   r4   r   r  r  r	   )r  r   r   Z	zo_scoresZ	f1_scoresrJ   rJ   rK   3test_cross_val_score_with_score_func_classification  s    
r7  c            	      C   s   t ddddd\} }t }t|| |}t|ddddd	gd
 t|| |dd}t|ddddd	gd
 t|| |dd}tdddddg}t||d
 tt}t|| ||d}t|ddddd	gd
 d S )Nr   r   r   r   )r   
n_featuresn_informativer   gGz?r5  gGz?gq=
ףp?r   r   r   r   g(\؇gzGIgGz&qg\(qg)\G)r    r-   r   r	   r   r   r%   r$   )	rU   rn   r   r   Z	r2_scoresZneg_mse_scoresZexpected_neg_mser   Z	ev_scoresrJ   rJ   rK   /test_cross_val_score_with_score_func_regression  s    r:  c               
   C   s  t  } | j}t|}| j}tdd}td}t|||d|dd\}}}|dksRtt|dd	 t|||d|dt	
|jd
d\}	}
}|	|kst||ksttdd}td}t|||d|dt	
|jd
d\}	}
}|	|kst||kstdd }t|}t|||d||d
d\}}
}t|dd t|dd t	t	t|d}t|||d|dd\}}}|dk snt|dks|td S )Nr   r   r   r   r6  )n_permutationsr   r   g?        rx   r   )r;  r   r   groupsr   c                 S   s"   | |k  | |k   | jd  S rE   )sumrN   )Zy_truer   rJ   rJ   rK   custom_score+  s    z,test_permutation_score.<locals>.custom_scored   )r;  r   r   r   g(\?g{Gz?r   rv   皙?)r"   r  r   r  r4   r   r   rk   r   r   r(  sizer%   modaranger   )r  rU   r   rn   r  r   r[   r   ZpvalueZscore_group_Zpvalue_groupZ
svm_sparseZ	cv_sparser?  scorerrJ   rJ   rK   test_permutation_score  s    
     



           rG  c                  C   st   t jdt jddd} t j| dd d f< t ddg| jd d }tdtd	t jd
fdt	 fg}t
|| | d S N   r  r   r   r   r   rx   r   mean)ZstrategyZmissing_valuesr   )r   rD  float64r   r   repeatrN   r:   r8   r|   r   rU   rn   prJ   rJ   rK   &test_permutation_test_score_allow_nans@  s    rO  c               	   C   s   t ddd} t dgd dgd  }tdd}d}tjt|d	 t|| | W 5 Q R X d
}tjt	|d	  t|| |dt 
did W 5 Q R X t|| |dt 
did d S )Nr@  r   r   r   rx   TZexpected_sample_weight#Expected sample_weight to be passedr   /sample_weight.shape == \(1,\), expected \(8,\)!r   r$  )r   rD  r   r   r   r   r   rk   r   r   r(  rU   rn   r   err_msgrJ   rJ   rK   &test_permutation_test_score_fit_paramsN  s    
$rU  c                  C   st   t jdt jddd} t j| dd d f< t ddg| jd d }tdtd	t jd
fdt	 fg}t
|| | d S rH  )r   rD  rK  r   r   rL  rN   r:   r8   r|   r   rM  rJ   rJ   rK   test_cross_val_score_allow_nans]  s    rV  c            	      C   s*  t ddgddgddgddgddgddgddgddgddgddgg
} t ddgddgddgddgddgddgddgddgddgddgg
}tdd	}ttd
d}ttdd}ttdd}t|| ||d}t|| ||d}t|| ||d}t|dddddg t|dddddg t|dddddg d S )Nr   r   r   r   rx   r   )Zn_neighborsmicro)ZaveragemacroZsamplesr   rv         ?gUUUUUU?      ?)r   r   r3   r%   r)   r   r   )	rU   rn   r   Zscoring_microZscoring_macroZscoring_samplesZscore_microZscore_macroZscore_samplesrJ   rJ   rK   test_cross_val_score_multilabelk  s4    >
r]  c               	   C   s  t dd\} }t }t }t|}|| |D ].\}}|| | ||  || | ||< q0t|| ||d}t	|| t|| |}t
|t
|kstt }t|| ||d}t
|t
|kst|  }||t|k9 }t|}t|||}t	t
|t
| ttdd| }t
|t
|ks*tG dd d}	tt t|| ||	 d W 5 Q R X tdd\} }d}
tjt|
d	" ttd
d| |dtdd W 5 Q R X d S )NTr   r   r   )Zn_initc                   @   s   e Zd ZdddZdS )z%test_cross_val_predict.<locals>.BadCVNc              	   s   s:   t dD ],}tddddgtddddd	gfV  qd S )
Nr   r   rx   r   r   r            )r   r   r   )rI   rU   rn   r=  irJ   rJ   rK   r     s    z+test_cross_val_predict.<locals>.BadCV.split)NN)r^   r_   r`   r   rJ   rJ   rJ   rK   BadCV  s   rb  zNumber of classes in training fold \(2\) does not match total number of classes \(3\). Results may not be appropriate for your use case.r   	liblinearsolverr   r   methodr   )r!   r   r-   r   Z
zeros_liker   rP   rV   r   r	   r   rk   r   copyZmedianr   r6   r   r   r   r"   r   RuntimeWarningr.   )rU   rn   r   r   Zpreds2r   r   predsZXsprb  r   rJ   rJ   rK   test_cross_val_predict  sF    

rk  c               	   C   sD  t dddd\} }ttdd| |dd}|jd	ks6ttd
d\} }ttdd| |dd}|jdksht| d d } |d d }d}tjt|d tt	 | |dt
dd W 5 Q R X td
d\} }tddd}t|| |dd}|jdkstt|}| | ||  } }d}tjt|d t|| |t
dddd W 5 Q R X d S )Nr   2   r   r   r   r   rc  rd  decision_functionrg  )rl  Tr      r   r@  zOnly 1 class/es in training fold, but 2 in overall dataset. This is not supported for decision_function with imbalanced folds. To fix this, use a cross-validation technique resulting in properly stratified foldsr   rf  r   Zovo)r   Zdecision_function_shape)i  -   zOutput shape \(599L?, 21L?\) of decision_function does not match number of classes \(7\) in fold. Irregular decision_function .*r   n_splitsr   rg  )r@   r   r.   rN   rk   r"   r   r   r   r1   r   r#   r4   r   Zargsort)rU   rn   rj  r   r   indr   rJ   rJ   rK   .test_cross_val_predict_decision_function_shape  sL              
rw  c                  C   sl   t dddd\} }ttdd| |dd}|jd	ks6ttd
d\} }ttdd| |dd}|jdkshtd S )Nr   rl  r   rm  rc  rd  r   ro  rl  r   Tr   rp  r@   r   r.   rN   rk   r"   rU   rn   rj  rJ   rJ   rK   *test_cross_val_predict_predict_proba_shape  s           r{  c                  C   sl   t dddd\} }ttdd| |dd}|jd	ks6ttd
d\} }ttdd| |dd}|jdkshtd S )Nr   rl  r   rm  rc  rd  predict_log_probaro  rx  Tr   rp  ry  rz  rJ   rJ   rK   .test_cross_val_predict_predict_log_proba_shape  s           r}  c            
      C   sb  t  } | j| j }}t|}t||d d d g}tddd}t|||}|jdksZt	t|||}|jdkstt	t|||}t
|jd t|||}t
|jd dd }t|d	}t|| | }t|d
}t||| }ttdd| | dd}ttdd|| dd}|d d d d tjf }dd }	t|	d	}t|||}t
|jd d S )Nr   Fr   )Zfit_interceptr   )rq  )rq  r   c                 S   s
   t | tS rQ   r   rh   rJ   rJ   rK   r   -  r   z4test_cross_val_predict_input_types.<locals>.<lambda>r   r   rc  rd  rn  ro  c                 S   s
   | j dkS )Nr   )r   rh   rJ   rJ   rK   r   D  r   )r"   r  r  r   r   r   r-   r   rN   rk   r
   r   r   r.   r   )
r  rU   rn   r   r   r   predictionsr   r   Zcheck_3drJ   rJ   rK   "test_cross_val_predict_input_types  sF    


r  c                     s   t t fg} z"ddlm}m} | ||f W n tk
r@   Y nX | D ]N\  tt }} fdd}fdd}t||d}t	|||dd qFd S )	Nr   r  c                    s
   t |  S rQ   r  rh   r  rJ   rK   r   X  r   z/test_cross_val_predict_pandas.<locals>.<lambda>c                    s
   t |  S rQ   r  rh   r  rJ   rK   r   Y  r   r
  r   r   )
r   r  r  r  r   r  rU   r   r   r   r  rJ   r  rK   test_cross_val_predict_pandasJ  s    
r  c                  C   s   t ddddddd\} }d|d< tddd}tdd}t|| |\}}t|| ||d	d
}||d  d dksttt||d  d d df dkstt||d  d d ddf dkstt||d  dkstt	|j
ddt|jdd d S )Nr@  r   r   rx   )r   r8  n_redundantr9  n_clusters_per_classr   rc  r   re  rs  r   ru  Zaxis   )decimal)r@   r.   r   rl   r   r   rk   r   r   r	   r>  r(  rN   )rU   rn   r   r   r   r   Z
yhat_probarJ   rJ   rK   !test_cross_val_predict_unbalanced^  s$    
	
&*r  c                  C   sf   t  } tjd}|dd}t| |d ddd}t|d d df | t| |d ddd}t|| d S )	Nr!  r@  r   r   rV   )rn   r   rg  r   r   )r|   r   randomRandomStateZrandr   r   )Zmock_classifierrngrU   Zy_hatZy_hat_probarJ   rJ   rK   test_cross_val_predict_y_nonet  s        r  c                  C   sX   t  } | j| j }}t }dtt|jd i}t||||dd}t	|t
d d S )Nr   r   r   )r%  r   )r"   r  r  r|   r   r   r&  rN   r   r
   r(  )r  rU   rn   r   r%  r~   rJ   rJ   rK   &test_cross_val_score_sparse_fit_params  s    r  c                  C   s  d} d}t | ddddddd\}}t| |d |  }dD ]}tjdd	6}t|||t|d
tddd|dd\}}}	}
}W 5 Q R X t|dkrt	d|d j
 |jdkst|	jdkst|
jdkst|jdkstt|tddd t|jddtddd t|	jddtddd |
jdks:t|jdksJttjdd	2}t|||t|| dtddd|d\}}}W 5 Q R X t|dkrt	d|d j
 t|| t||	 q:d S )Nr   r   rx   r   r   r   r8  r9  r  r   r  r   FTTr-  rs  皙?r   r   )r   rG   r?   Zreturn_timesUnexpected warning: %r)r   r   r   r  ffffff?rK  rt  r   )r   rG   r?   )r@   rD   r/  r0  r   r   r   linspacer   RuntimeErrormessagerN   rk   r
   r	   rJ  r  rB   )r   rt  rU   rn   r   shuffle_trainwrG   train_scorestest_scoresZ	fit_timesZscore_timesZtrain_sizes2Ztrain_scores2Ztest_scores2rJ   rJ   rK   test_learning_curve  sl    
	



r  c               
   C   s   t dddddddd\} }td}t|| d dtdd	d
d\}}}t|tddd
 t|jddtdd	d
 t|jddtdd	d
 d S )Nr   rx   r   r   r  r   r   r  r   r   )rn   r   rG   r  r  )r@   rD   r   r   r  r
   r	   rJ  rU   rE  r   rG   r  r  rJ   rJ   rK    test_learning_curve_unsupervised  s(    
	    r  c               	   C   sz   t dddddddd\} }td}tj}t t_zt|| |ddd\}}}W 5 tj }tj  |t_X d	|ksvtd S )
Nr   rx   r   r   r  r   r   )r   verbosez[learning_curve])	r@   rD   sysstdoutr;   getvaluecloser   rk   )rU   rn   r   Z
old_stdoutoutrG   r  r  rJ   rJ   rK   test_learning_curve_verbose  s0    
	    

r  c               	   C   sL   t dddddddd\} }td}tt t|| |dd W 5 Q R X d S )Nr   rx   r   r  T)exploit_incremental_learningr@   rD   r   r   r   r   rU   rn   r   rJ   rJ   rK   5test_learning_curve_incremental_learning_not_possible  s    

r  c                  C   s   t dddddddd\} }td}dD ]v}t|| |dd	td
dd|d\}}}t|tddd t|jddtddd t|jddtd
dd q&d S )Nr   rx   r   r   r  r   r  r   Tr  r   r   )r   r  rG   r?   r  r  r@   rb   r   r   r  r
   r	   rJ  )rU   rn   r   r  rG   r  r  rJ   rJ   rK   (test_learning_curve_incremental_learning  s.    
		r  c                  C   s   t dddddddd\} }td}t|| d ddtd	d
dd\}}}t|tddd t|jddtdd
d t|jddtd	d
d d S )Nr   rx   r   r   r  r   r   Tr  r   r   )rn   r   r  rG   r  r  r  r  rJ   rJ   rK   5test_learning_curve_incremental_learning_unsupervised  s*    
	r  c            
   	   C   s   t dddddddd\} }tddd}tdd d	d
}t|| ||ddd\}}}t|| |d|d	d\}}}	t|| t|jdd|jdd t|jdd|	jdd d S )Nr   rx   r   r   r  rA  r   r   Fmax_itertolr?   r   T)rG   r   r  )r   rG   r  r  )r@   r   r  r0   r   r
   r	   rJ  )
rU   rn   rG   r   train_sizes_inctrain_scores_inctest_scores_inctrain_sizes_batchtrain_scores_batchtest_scores_batchrJ   rJ   rK   <test_learning_curve_batch_and_incremental_learning_are_equal6  sH    
		

 

 
r  c               	   C   s  t dddddddd\} }td}tt t|| |dddgd W 5 Q R X tt t|| |dd	d
gd W 5 Q R X tt t|| |dddgd W 5 Q R X tt t|| |dddgd W 5 Q R X tt t|| |dddgd W 5 Q R X d S )Nr   rx   r   r   r  r   r   r   rG   r<  r   r  g?   r  r  rJ   rJ   rK   0test_learning_curve_n_sample_range_out_of_bounds]  s(    
	    r  c                  C   st   t dddddddd\} }td}d}tjt|d( t|| |dtdd	dd
\}}}W 5 Q R X t|ddg d S )Nr   rx   r   r   r  zzRemoved duplicate entries from 'train_sizes'. Number of ticks will be less than the size of 'train_sizes': 2 instead of 3.r   gQ?r   r  )	r@   rD   r   r   ri  r   r   r  r
   )rU   rn   r   r   rG   rE  rJ   rJ   rK   1test_learning_curve_remove_duplicate_sample_sizest  s*    
	    r  c               
   C   s   t dddddddd\} }td}tdd}t|| ||td	d
dd\}}}t|tddd t|jddtdd
d t|jddtd	d
d d S )Nr   rx   r   r   r  r   r   rs  r  r   r   r  r  r  )	r@   rD   r   r   r   r  r
   r	   rJ  )rU   rn   r   r   rG   r  r  rJ   rJ   rK   (test_learning_curve_with_boolean_indices  s*    
	
    r  c                  C   s  t ddgddgddgddgd	d
gddgddgddgddgddgddgd	d
gddgddgddgg} t dddddddddddddddg}t dddddddddddddddg}tdd dd}tdd}t|| ||dt ddd|ddd	\}}}t|jddt dddg t|jddt dddg t	t
( t|| ||dt ddd|d d! W 5 Q R X t|| ||dt ddd|dddd"
\}}	}
t|	jdd|jdd t|
jdd|jdd d S )#Nrx   r   r   r   r   r^  r_  r`     r                       r   	   r   Fr  rs  g333333?r   T)r   r   rG   r=  r?   r   r  r[  gIq?r\  r   )r   r   rG   r=  r   )r   r   rG   r=  r?   r   r  )r   r   r0   r   r   r  r	   rJ  r   r   r   )rU   rn   r=  r   r   r  r  r  r  r  r  rJ   rJ   rK    test_learning_curve_with_shuffle  s    ((

 
 
 

 
r  c               
   C   s   t ddd} t dgd dgd  }tdd}d}tjt|d	 t|| |d
d W 5 Q R X d}tjt	|d	" t|| |d
dt 
did W 5 Q R X t|| |d
dt 
did d S )Nr@  r   r   r   rx   TrP  rQ  r   r   r   z/sample_weight.shape == \(1,\), expected \(2,\)!r   )r   r%  )r   rD  r   r   r   r   r   rk   r   r   r(  rS  rJ   rJ   rK   test_learning_curve_fit_params  s,    
        r  c                  C   s   t dddddddd\} }tddg}d}tjt|d	& t|| |d
dtddddd W 5 Q R X d}tjt|d	2 t|| |d
dtdddddtd
id W 5 Q R X t|| |d
dtdddddtdid d S )Nr   rx   r   r   r  r   r   z9Expected fit parameter\(s\) \['sample_weight'\] not seen.r   r   Tr  r   r   r   )r   r  rG   r   z2Fit parameter sample_weight has length 3; expected)r   r  rG   r   r%  )	r@   rb   r   r   rk   r   r   r  r(  )rU   rn   r   rT  rJ   rJ   rK   3test_learning_curve_incremental_learning_fit_params  sV    
	
r  c               	   C   s   t dddddddd\} }tddd}tjdd }tt | |d|dd	\}}W 5 Q R X t|dkrxtd
|d j	 t
|jdd| t
|jddd|  d S )Nr   rx   r   r  r   Tr-  rw   
param_nameparam_ranger   r  r  )r@   r   r  r/  r0  r   ru   r   r  r  r	   rJ  )rU   rn   r  r  r  r  rJ   rJ   rK   test_validation_curve/  s.    
	r  c               	   C   sF   t dddddddd\} }tddd}tt | |d|dd\}}d S )Nr   rx   r   r  r   rw   r  )r@   r   r  r   ry   )rU   rn   r  rE  rJ   rJ   rK   %test_validation_curve_clone_estimatorJ  s$    

r  c               
   C   s   d} d}t ddd\}}ttddd||dddd	d	gt|| d
d}ttt|dd d f d  ttddd||dddd	d	gt|ddd}ttt|dd d f d  ttddd||dddd	d	gt|dd}tt	|t	| d S )Nr@  r   r   r   r   r   r   r  rA  r  r  )r   r   rx   r   r   Trt  r?   rs  )
r@   r   r4   rB   r	   r   Zvsplithstackr   r   )r   rt  rU   rn   Zscores1Zscores2Zscores3rJ   rJ   rK   +test_validation_curve_cv_splits_consistency`  s<    


"


"


r  c                  C   s   t ddd} t dgd dgd  }tdd}d}tjt|d	  t|| |d
dddgdd W 5 Q R X d}tjt	|d	, t|| |d
dddgddt 
did W 5 Q R X t|| |d
dddgddt 
did d S )Nr@  r   r   r   rx   TrP  rQ  r   Z	foo_paramr   r   r   )r  r  r   rR  r   )r  r  r   r%  )r   rD  r   r   r   r   r   rk   r   r   r(  rS  rJ   rJ   rK    test_validation_curve_fit_params  sB    
		r  c                  C   sv   t jd} t d}| | t|ds.ttt |ddrDtd|d< t|drZttt |dfdrrtd S )Nr   r@     )	r   r  r  rD  r?   r   rk   deleter  )r  rN  rJ   rJ   rK   test_check_is_permutation  s    

r  c                  C   sj   t dddddd\} }t| }t|}ttdd}t|| |dd	}t|||dd	}| }t|| d S )
Nr   rx   FT)r   n_labelsZallow_unlabeledZreturn_indicatorr   r   r   r   r   )rA   r   r>   r4   r   Ztoarrayr	   )rU   rn   r   Zy_sparseZclassifrj  Zpreds_sparserJ   rJ   rK   (test_cross_val_predict_sparse_prediction  s    
r  c           
   	   C   s   t ddd}|jdkr6|dkr(t|fn
t|df}n|j}t|}|||D ]6\}}t| || || } t	| ||| ||< qR||d |d |
dfD ]}	tt| ||	||d| qd	S )
z@Helper for tests of cross_val_predict with binary classificationr   Fr  rx   rn  r   strrf  N)r   r   r   rN   r   r  r   r=   rP   getattrastyper   r   )
r   rU   rn   rg  r   	exp_shapeexpected_predictionsr   r   tgrJ   rJ   rK   check_cross_val_predict_binary  s    
 
 r  c              	   C   s   t ddd}ttjj}||dd}tjt|tt|f|| tjd}tj|dd\}}	|	||	D ]P\}
}t
| ||
 |	|
 } t| ||| }t|	|
 }||t||< qj||d	 |d
 |dfD ]}tt| ||||d| qdS )zDHelper for tests of cross_val_predict with multiclass classificationr   Fr  r   rn  r|  r   r  TZreturn_inverserx   r   r  rf  N)r   r   finforK  minr)  r   rj   r   r   r=   rP   r  ix_r  r   r   )r   rU   rn   rg  r   	float_mindefault_valuesr  rE  y_encr   r   
fold_predsZ
i_cols_fitr  rJ   rJ   rK   "check_cross_val_predict_multiclass  s,       r  c                    s  t ddd}ttjj}||dd} jd }g }t|D ]`}	tt dd|	f }
|
dkrt|d	krtt|f}nt||
f}|	tj
||| tjd
 q< fddt jd D }tj|dd}|||D ]\}}t| || || } t| ||| }t|D ]^}	t|| dd|	f }||	 jdkrN||	 ||	 |< nt||}||	 ||	 |< qqԈ  d  d  dfD ]T}t| ||||d}t|t|ksttt|D ]}t|| ||  qqdS )zCheck the output of cross_val_predict for 2D targets using
    Estimators which provide a predictions as a list with one
    element per class.
    r   Fr  r   r  rx   Nr   rn  r  c                    s8   g | ]0}t j d d |f ddd d d t jf qS )NTr  rx   )r   r   r   )r   ra  rn   rJ   rK   
<listcomp>   s   z6check_cross_val_predict_multilabel.<locals>.<listcomp>r  r  rf  )r   r   r  rK  r  rN   r   r   rj   r   r)  Zconcatenater   r=   rP   r  r   r   r  r  r   rk   r   )r   rU   rn   rg  r   r  r  	n_targetsZexpected_predsZi_colZn_classes_in_labelr  Z
y_enc_colsr  r   r   r  Z	fold_colsidxr  Zcv_predict_outputra  rJ   r  rK   "check_cross_val_predict_multilabel  sD    

r  c                 C   s,   t ddd\}}dD ]}t| ||| qd S )Nr   r   )r   r   rn  r   r|  )r@   r  )r   rU   rn   rg  rJ   rJ   rK   *check_cross_val_predict_with_method_binary9  s    r  c                 C   sB   t  }|j|j }}t||dd\}}dD ]}t| ||| q*d S )Nr   r   r  )r"   r  r  r?   r  )r   r  rU   rn   rg  rJ   rJ   rK   .check_cross_val_predict_with_method_multiclassA  s
    r  c                   C   s    t tdd ttdd d S )Nrc  rd  )r  r.   r  rJ   rJ   rJ   rK   "test_cross_val_predict_with_methodI  s    r  c                  C   sN   t  } | j| j }}t||dd\}}dD ]}tddd}t|||| q*d S )Nr   r   r  Zlog_lossr   )Zlossr   )r"   r  r  r?   r/   r  )r  rU   rn   rg  r   rJ   rJ   rK   &test_cross_val_predict_method_checkingP  s    r  c                  C   s`   t  } | j| j }}t||dd\}}ttddddddgid	d
}dD ]}t|||| qHd S )Nr   r   r!  rc  r  r   r  rx   r   r   r  )r"   r  r  r?   rC   r.   r  )r  rU   rn   r   rg  rJ   rJ   rK   /test_gridsearchcv_cross_val_predict_with_method[  s    
 
 r  c                  C   sL   d} d}t | d|ddd\}}ttddd	}d
D ]}t||||d q2d S )Nr@  r   r   r   r!  r   r  r   r8  r   rc  r   )re  r   )r   rn  ro  )rA   r>   r.   r  )Zn_sampr   rU   rn   r   rg  rJ   rJ   rK   1test_cross_val_predict_with_method_multilabel_ovrf  s        
r  c                   @   s   e Zd Zdd ZdS )RFWithDecisionFunctionc                 C   s2   |  |}d}t|ts t|dd |D }|S )Nz?This helper should only be used on multioutput-multiclass tasksc                 S   s.   g | ]&}|j d  dkr&|dddf n|qS )rx   r   Nr   r   )r   rN  rJ   rJ   rK   r  |  s     z<RFWithDecisionFunction.decision_function.<locals>.<listcomp>)r   r   rl   rk   )rI   rU   ZprobsmsgrJ   rJ   rK   rn  x  s
    
z(RFWithDecisionFunction.decision_functionN)r^   r_   r`   rn  rJ   rJ   rJ   rK   r  t  s   r  c               
   C   s   d} t dd| ddd\}}|d d df  |d d df 7  < d	D ]>}tddd
}t   td t||||d W 5 Q R X qBd S )Nr   r@  r   r   r!  r  r   rx   r   r|  rn  Zn_estimatorsr   ignorero  )rA   r  r/  r0  simplefilterr  )r   rU   rn   rg  r   rJ   rJ   rK   0test_cross_val_predict_with_method_multilabel_rf  s        
$

r  c                  C   s   t jd} | jdddd}t ddddddddddddddg}tdd}d	D ]0}t  td
 t	|||| W 5 Q R X qPd S )Nr   rx   )r  r   rB  r   r   rc  rd  r  r  )
r   r  r  normalr   r.   r/  r0  r  r  )r  rU   rn   r   rg  rJ   rJ   rK   -test_cross_val_predict_with_method_rare_class  s    &


r  c               
   C   s   t jd} | jdddd}t ddgddgddgddgddgg}dD ]>}tddd}t   td	 t	||||d
 W 5 Q R X qHd S )Nr   rx   )r   r   r  r   )r   r|  r   r  r  ro  )
r   r  r  r  r   r  r/  r0  r  r  )r  rU   rn   rg  r   rJ   rJ   rK   ;test_cross_val_predict_with_method_multilabel_rf_rare_class  s    (

r  c                 C   s   t t||g}t||}|| |D ]|\}}	|| | ||  || |	 }
|dkrlt t|	|f}nt t|	|ft |jj	}|
|d d |j
f< |||	< q(|S )Nr   )r   r  r   r  r   rP   r)  r  r  r  Zclasses_)rU   rn   r   classesr   rg  r  funcr   r   Zexpected_predictions_Zexp_pred_testrJ   rJ   rK   get_expected_predictions  s    

 
r  c                  C   s  t ddd} t dd tdD }d}tdd}td	d}t }d
ddg}|D ]}tdd}t|| |||d}	t	| |||||}
t
|
|	 t|| |||d}	t	| |||||}
t
|
|	 tt tdddd}t|| |||d}	||}t	| |||||}
t
|
|	 qVd S )NrI  r@  r   c                 S   s   g | ]}|d  qS )r   rJ   )r   rd   rJ   rJ   rK   r    s     z7test_cross_val_predict_class_subset.<locals>.<listcomp>r   r   rs  r   rn  r   r|  rc  rd  rf  r   r   )r   rD  r   r   r   r   r9   r.   r   r  r	   r?   rL  Zfit_transform)rU   rn   r  Zkfold3Zkfold4lemethodsrg  r   r~  r  rJ   rJ   rK   #test_cross_val_predict_class_subset  sR    



     
     

     r  c                     s   t  } | j| j }}t }tjddd}|d |  tj	|j
tjdtj	|j
ddtjd zJt||| fddd tt t|||fddd W 5 Q R X W 5 d	\ td
D ]8}zt|j
 W  qW q tk
r   td Y qX qX d S )NwbF)moder  s   Hello world!!!!!r  rJ   r)rN   r  r  )NNr   r   c                    s    S rQ   rJ   r   rU   rn   )r[   rJ   rK   r     r   z#test_score_memmap.<locals>.<lambda>r   c                    s    S rQ   rJ   r  )r   rJ   rK   r     r   )r"   r  r  r|   tempfileNamedTemporaryFilewriter  r   ZmemmapnamerK  r   osunlinkWindowsErrorr   r   r   r   r   )r  rU   rn   r   tfrE  rJ   )r[   r   rK   test_score_memmap  s&    
&
r  c                     s   t t fg} z"ddlm}m} | ||f W n tk
r@   Y nX | D ]^\ t }|j|j }} || }} fdd}fdd}	t	||	d}
t
|
|| qFd S )Nr   r  c                    s
   t |  S rQ   r  rh   r  rJ   rK   r     r   z4test_permutation_test_score_pandas.<locals>.<lambda>c                    s
   t |  S rQ   r  rh   r  rJ   rK   r     r   r
  )r   r  r  r  r   r  r"   r  r  r   r   )r  r  r  r  rU   rn   r  r  r  r  r   rJ   r  rK   "test_permutation_test_score_pandas  s    
r  c               
   C   s8  t t j} tdd}td}| |d t d d dd d g	}ddi}tjtdd t	|| W 5 Q R X t
d	}tjt|d t| |d
dd W 5 Q R X tjt|d t| |d
dd W 5 Q R X tjt|d t| ||d
dd W 5 Q R X tjt|d  t| ||dt jgd
dd W 5 Q R X |  dks4td S )Nrx   r   r  r   r   r   z%Failing classifier failed as requiredr   zerror_score must be the string 'raise' or a numeric value. (Hint: if using 'raise', please make sure that it has been spelled correctly.)r   zunvalid-stringr   r   Z	parameter)r  r  r   r   r<  )r   FAILING_PARAMETERr   rD  r(  r   r   r   r   r   reescaper   r   r   r   r[   rk   )failing_clfrU   rn   fit_and_score_argsfit_and_score_kwargsr   rJ   rJ   rK   test_fit_and_score_failing  s6    


r  c                  C   sx   t ddd\} }tddd}tt | \}}|| |t ||dg}dddd d	d
}t||}|d |d ksttd S )Nr   r   r   r   r   r@  r  )r  r  T)
parametersr%  Zreturn_parametersr  )r@   r4   nextr   r   r   r   rk   )rU   rn   r   r   r   r  r  resultrJ   rJ   rK   test_fit_and_score_workingF  s    
r  c                   @   s*   e Zd ZdddZd	ddZd
ddZdS )DataDependentFailingClassifierNc                 C   s
   || _ d S rQ   max_x_value)rI   r  rJ   rJ   rK   rL   V  s    z'DataDependentFailingClassifier.__init__c                 C   s&   || j k }|r"td| dd S )NzClassifier fit failed with z values too high)r  r>  r   )rI   rU   rn   Znum_values_too_highrJ   rJ   rK   rP   Y  s
    
z"DataDependentFailingClassifier.fitc                 C   s   dS )Nr<  rJ   rY   rJ   rJ   rK   r[   `  s    z$DataDependentFailingClassifier.score)N)N)NN)r^   r_   r`   rL   rP   r[   rJ   rJ   rJ   rK   r  U  s   

r  r   c              	   C   s   t dd}tdd}td}|||g}d| d}d}tjd	|d
  d| tjd}tjt	|d t
|| W 5 Q R X d S )Nr`  r  rx   r   r  r   r  z8ValueError: Classifier fit failed with 1 values too highzh2 fits failed.+total of 3.+The score on these train-test partitions for these parameters will be set to r   z.+flagsr   )r  r   rD  r(  r  compileDOTALLr   r   r   r   )r   r  rU   rn   cross_validate_argscross_validate_kwargsindividual_fit_error_messager   rJ   rJ   rK   -test_cross_validate_some_failing_fits_warningd  s    



r'  c              	   C   sv   t t j}tdd}td}|||g}d| d}d}tjd| tjd}tj	t
|d	 t|| W 5 Q R X d S )
Nrx   r   r  r_  r  z1ValueError: Failing classifier failed as requiredz4All the 7 fits failed.+your model is misconfigured.+r   r   )r   r  r   rD  r(  r  r"  r#  r   r   r   r   )r   r  rU   rn   r$  r%  r&  r   rJ   rJ   rK   *test_cross_validate_all_failing_fits_error~  s    



r(  c                 C   s   t |d S rQ   )r   )r   rU   rn   	error_msgrJ   rJ   rK   _failing_scorer  s    r*  zignore:lbfgs failed to converger   c              	   C   s   t dd\}}tdd||}d}tt|d}| dkrhtjt|d t|||d	|| d
 W 5 Q R X nBd|  }tj	t
|d$ t|||d	|| d
}t||  W 5 Q R X d S )NTr   r   r  "This scorer is supposed to fail!!!r)  r   r   r   )r   r   r   r   )r"   r.   rP   r   r*  r   r   r   r   r   r   r   )r   rU   rn   r   r)  failing_scorerwarning_msgr   rJ   rJ   rK   #test_cross_val_score_failing_scorer  s4              r0  r   TFwith_multimetricc              
   C   s  t dd\}}tdd||}d}tt|d}|rJtt}|||d}	n|}	| dkrtjt	|d	 t
|||d
|	|| d W 5 Q R X nd|  }
tjt|
d	b t
|||d
|	|| d}|D ]@}d|krd|kr|| D ]}t|tstqqt|| |  qW 5 Q R X d S )NTr   r   r+  r,  r-  )Zscore_1Zscore_2Zscore_3r   r   r   )r   r   r   r   r   r   Z_score_2)r"   r.   rP   r   r*  r%   r+   r   r   r   r   r   r   r   rX   rk   r   )r   r   r1  rU   rn   r   r)  r.  Znon_failing_scorerr   r/  r   rq   ra  rJ   rJ   rK   "test_cross_validate_failing_scorer  sP    	r2  c                 C   s   dS )Ng;pΈ^@rJ   )ra  jr   rJ   rJ   rK   three_params_scorer  s    r4  z:train_score, scorer, verbose, split_prg, cdt_prg, expected)rx   r   r   rx   zS\[CV\] END .................................................... total time=   0.\ds)Zsc1Zsc2ze\[CV 2/3\] END  sc1: \(train=3.421, test=3.421\) sc2: \(train=3.421, test=3.421\) total time=   0.\dsr   zW\[CV 2/3; 1/1\] END ....... sc1: \(test=3.421\) sc2: \(test=3.421\) total time=   0.\dsc              	   C   s   t ddd\}}tddd}	tt |\}
}|	||||
||d d g	}|||d}t|| |  \}}|d}t|dkrt	||d	 st
nt	||d st
d S )
Nr   r   r   r   r   )r   Zsplit_progressZcandidate_progress
r   rx   )r@   r4   r  r   r   r   Z
readouterrr   r  r   rk   )Zcapsysr   rF  r  Z	split_prgZcdt_prgexpectedrU   rn   r   r   r   r  r  r  rE  ZoutlinesrJ   rJ   rK   test_fit_and_score_verbosity  s    #

r8  c               	   C   sF   d} dd }d d d |g}t jt| d t|dtji W 5 Q R X d S )Nz&scoring must return a number, got Nonec                 S   s   d S rQ   rJ   )r   ZX_testrJ   rJ   rK   two_params_scorer3	  s    z%test_score.<locals>.two_params_scorerr   r   )r   r   r   r   r   r   )r   r9  r  rJ   rJ   rK   r   0	  s
    r   c                  C   sn   dd } t dddd\}}tdd}||| t|||d| d	}d
dddg}|D ]}d||ksRtqRd S )Nc                 S   s2   |  |}t||}|d |d |d |d dS )N)r   r   r5  )rx   r   )rx   rx   )tnfpfntp)rV   r'   )r   rU   rn   r   cmrJ   rJ   rK   r   <	  s    

zPtest_callable_multimetric_confusion_matrix_cross_validate.<locals>.custom_scorer(   r   r!  )r   r8  r   r   r   )r   r   r:  r;  r<  r=  ztest_{})r@   r5   rP   r   r   rk   )r   rU   rn   r   r   Zscore_namesr
  rJ   rJ   rK   9test_callable_multimetric_confusion_matrix_cross_validate;	  s    
r@  c                  C   s&   t dd\} }tt | |ddd dS )z^Check that regressors with partial_fit is supported.

    Non-regression test for #22981.
    r!  r   Tr   )r  r   N)r    r   r7   )rU   rn   rJ   rJ   rK   *test_learning_curve_partial_fit_regressorsK	  s    rA  )ra   r  r  r  r  r/  	functoolsr   timer   r   Znumpyr   Zscipy.sparser   r   Zsklearn.exceptionsr   Z)sklearn.model_selection.tests.test_searchr   Zsklearn.utils._testingr   r	   r
   r   Zsklearn.utils._mockingr   r   Zsklearn.utils.validationr   Zsklearn.model_selectionr   r   r   r   r   r   r   r   r   r   r   r   r   r   Z#sklearn.model_selection._validationr   r   r   Zsklearn.datasetsr    r!   r"   r#   Zsklearn.metricsr$   r%   r&   r'   r(   r)   r*   r+   r,   Zsklearn.linear_modelr-   r.   r/   r0   r1   Zsklearn.ensembler2   Zsklearn.neighborsr3   Zsklearn.svmr4   r5   Zsklearn.clusterr6   Zsklearn.neural_networkr7   Zsklearn.imputer8   Zsklearn.preprocessingr9   Zsklearn.pipeliner:   ior;   Zsklearn.baser<   r=   Zsklearn.multiclassr>   Zsklearn.utilsr?   r@   rA   Z$sklearn.model_selection.tests.commonrB   rC   r  	NameErrorrD   rb   ru   ry   r|   r(  rU   r   r   rn   r   r&  r   r   r   r   r   r   r   r   r  markfilterwarningsr  r  r  r*  r1  r4  r7  r:  rG  rO  rU  rV  r]  rk  rw  r{  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  Zparametrizer   r'  r(  r*  r0  r2  r4  r8  r   r@  rA  rJ   rJ   rJ   rK   <module>   s  
W
*
?/0I

#B845

	='N0.&4)

)



=

 
