U
    3dL                  	   @   s:  d Z ddlmZmZmZ ddlmZmZmZ ddl	Z	ddl
ZddlmZ ddlmZmZmZmZ dd	lmZ e	eZed
dddZeddddZeddddeddddeddddfZd0ddZdd Zd1d d!Zddd"dded#d$ed%d&fddd'd(d)Zd2d*d+Z d,ddd"ded#d$ed%d&fdd-d.d/Z!dS )3zLabeled Faces in the Wild (LFW) dataset

This dataset is a collection of JPEG pictures of famous people collected
over the internet, all details are available on the official website:

    http://vis-www.cs.umass.edu/lfw/
    )listdirmakedirsremove)joinexistsisdirN)Memory   )get_data_home_fetch_remoteRemoteFileMetadata
load_descr   )Bunchzlfw.tgzz.https://ndownloader.figshare.com/files/5976018Z@055f7d9c632d7370e6fb4afc7468d40f970c34a80d4c6f50ffec63f5a8d536c0)filenameurlZchecksumzlfw-funneled.tgzz.https://ndownloader.figshare.com/files/5976015Z@b47c8422c8cded889dc5a13418c4bc2abbda121092b3533a83306f90d900100apairsDevTrain.txtz.https://ndownloader.figshare.com/files/5976012Z@1d454dada7dfeca0e7eab6f65dc4e97a6312d44cf142207be28d688be92aabfapairsDevTest.txtz.https://ndownloader.figshare.com/files/5976009Z@7cb06600ea8b2814ac26e946201cdb304296262aad67d046a16a7ec85d0ff87c	pairs.txtz.https://ndownloader.figshare.com/files/5976006Z@ea42330c62c92989f9d7c03237ed5d591365e89b3e649747777b70e692dc1592Tc           
      C   s  t | d} t| d}t|s$t| tD ]D}t||j}t|s(|r`td|j t	||d q(t
d| q(|rt|d}t}nt|d}t}t|s
t||j}t|s|rtd|j t	||d nt
d| d	d
l}	td| |	|dj|d t| ||fS )z0Helper function to download any missing LFW data)	data_homelfw_homezDownloading LFW metadata: %s)dirnamez%s is missingZlfw_funneledZlfwz!Downloading LFW data (~200MB): %sr   Nz$Decompressing the data archive to %szr:gz)path)r
   r   r   r   TARGETSr   loggerinfor   r   IOErrorFUNNELED_ARCHIVEARCHIVEtarfiledebugopen
extractallr   )
r   funneleddownload_if_missingr   targetZtarget_filepathdata_folder_patharchivearchive_pathr    r)   9/tmp/pip-unpacked-wheel-zrfo1fqw/sklearn/datasets/_lfw.py_check_fetch_lfwJ   s8    




r+   c                 C   s  zddl m} W n tk
r,   tdY nX tddtddf}|dkrP|}ntdd t||D }|\}}|j|j |jpd }|j|j |jpd }	|dk	rt	|}t
|| }t
||	 }	t| }
|stj|
||	ftjd	}ntj|
||	d
ftjd	}t| D ]\}}|d dkr0td|d |
 ||}||j|j|j|jf}|dk	rl||	|f}tj|tjd	}|jdkrtd| |d }|s|jdd}|||df< q|S )zInternally used to load imagesr   )ImagezThe Python Imaging Library (PIL) is required to load data from jpeg files. Please refer to https://pillow.readthedocs.io/en/stable/installation.html for installing PIL.   Nc                 s   s   | ]\}}|p|V  qd S )Nr)   ).0sZdsr)   r)   r*   	<genexpr>   s     z_load_imgs.<locals>.<genexpr>r	   Zdtype   i  zLoading face #%05d / %05dzLFailed to read the image file %s, Please make sure that libjpeg is installedg     o@r   )Zaxis.)ZPILr,   ImportErrorslicetuplezipstopstartstepfloatintlennpzerosZfloat32	enumerater   r    r!   ZcropresizeZasarrayndimRuntimeErrorZmean)
file_pathsslice_colorr@   r,   Zdefault_sliceZh_sliceZw_slicehwn_facesfacesi	file_pathZpil_imgfacer)   r)   r*   
_load_imgsu   sT    
	

rM   Fc                    s   g g  }}t t| D ]h}t| | t s.q fddt t D }t|}	|	|kr|dd}||g|	  || qt|}
|
dkrtd| t	|}t
||}t||||}t|
}tjd| || ||  }}|||fS )z~Perform the actual data loading for the lfw people dataset

    This operation is meant to be cached by a joblib wrapper.
    c                    s   g | ]}t  |qS r)   )r   )r.   fZfolder_pathr)   r*   
<listcomp>   s     z%_fetch_lfw_people.<locals>.<listcomp>_ r   z*min_faces_per_person=%d is too restrictive*   )sortedr   r   r   r<   replaceextend
ValueErrorr=   uniqueZsearchsortedrM   ZarangerandomZRandomStateshuffle)r&   rD   rE   r@   min_faces_per_personZperson_namesrC   Zperson_namepathsZ
n_picturesrH   target_namesr%   rI   indicesr)   rO   r*   _fetch_lfw_people   s.    	



r_   g      ?F      N      )r   r#   r@   r[   rE   rD   r$   
return_X_yc                 C   s   t | ||d\}}	td| t|ddd}
|
t}||	||||d\}}}|t|d}td}|rr||fS t	|||||d	S )
a  Load the Labeled Faces in the Wild (LFW) people dataset (classification).

    Download it if necessary.

    =================   =======================
    Classes                                5749
    Samples total                         13233
    Dimensionality                         5828
    Features            real, between 0 and 255
    =================   =======================

    Read more in the :ref:`User Guide <labeled_faces_in_the_wild_dataset>`.

    Parameters
    ----------
    data_home : str, default=None
        Specify another download and cache folder for the datasets. By default
        all scikit-learn data is stored in '~/scikit_learn_data' subfolders.

    funneled : bool, default=True
        Download and use the funneled variant of the dataset.

    resize : float or None, default=0.5
        Ratio used to resize the each face picture. If `None`, no resizing is
        performed.

    min_faces_per_person : int, default=None
        The extracted dataset will only retain pictures of people that have at
        least `min_faces_per_person` different pictures.

    color : bool, default=False
        Keep the 3 RGB channels instead of averaging them to a single
        gray level channel. If color is True the shape of the data has
        one more dimension than the shape with color = False.

    slice_ : tuple of slice, default=(slice(70, 195), slice(78, 172))
        Provide a custom 2D slice (height, width) to extract the
        'interesting' part of the jpeg files and avoid use statistical
        correlation from the background.

    download_if_missing : bool, default=True
        If False, raise a IOError if the data is not locally available
        instead of trying to download the data from the source site.

    return_X_y : bool, default=False
        If True, returns ``(dataset.data, dataset.target)`` instead of a Bunch
        object. See below for more information about the `dataset.data` and
        `dataset.target` object.

        .. versionadded:: 0.20

    Returns
    -------
    dataset : :class:`~sklearn.utils.Bunch`
        Dictionary-like object, with the following attributes.

        data : numpy array of shape (13233, 2914)
            Each row corresponds to a ravelled face image
            of original size 62 x 47 pixels.
            Changing the ``slice_`` or resize parameters will change the
            shape of the output.
        images : numpy array of shape (13233, 62, 47)
            Each row is a face image corresponding to one of the 5749 people in
            the dataset. Changing the ``slice_``
            or resize parameters will change the shape of the output.
        target : numpy array of shape (13233,)
            Labels associated to each face image.
            Those labels range from 0-5748 and correspond to the person IDs.
        target_names : numpy array of shape (5749,)
            Names of all persons in the dataset.
            Position in array corresponds to the person ID in the target array.
        DESCR : str
            Description of the Labeled Faces in the Wild (LFW) dataset.

    (data, target) : tuple if ``return_X_y`` is True
        A tuple of two ndarray. The first containing a 2D array of
        shape (n_samples, n_features) with each row representing one
        sample and each column representing the features. The second
        ndarray of shape (n_samples,) containing the target samples.

        .. versionadded:: 0.20
    r   r#   r$   z Loading LFW people faces from %s   r   locationcompressverbose)r@   r[   rE   rD   lfw.rst)dataZimagesr%   r]   DESCR)
r+   r   r    r   cacher_   reshaper<   r   r   )r   r#   r@   r[   rE   rD   r$   rd   r   r&   m	load_funcrI   r%   r]   Xfdescrr)   r)   r*   fetch_lfw_people   s4    ^  

    ru   c              
   C   s  t | d}dd |D }W 5 Q R X dd |D }t|}tj|td}	t }
t|D ]\}}t|dkrd|	|< |d t|d d f|d t|d	 d ff}nZt|d
krd|	|< |d t|d d f|d	 t|d d ff}ntd|d |f t|D ]l\}\}}zt||}W n& t	k
rH   t|t
|d}Y nX ttt|}t||| }|
| qqVt|
|||}t|j}|d}|dd	 |d|d	  ||_||	tddgfS )z}Perform the actual data loading for the LFW pairs dataset

    This operation is meant to be cached by a joblib wrapper.
    rbc                 S   s   g | ]}|   d qS )	)decodestripsplit)r.   lnr)   r)   r*   rP   w  s     z$_fetch_lfw_pairs.<locals>.<listcomp>c                 S   s   g | ]}t |d kr|qS )r   )r<   )r.   slr)   r)   r*   rP   x  s      r1   r2   r	   r   r      zinvalid line %d: %rzUTF-8zDifferent personszSame person)r!   r<   r=   r>   r;   listr?   rW   r   	TypeErrorstrrT   r   appendrM   shapepopinsertarray)index_file_pathr&   rD   rE   r@   Z
index_fileZsplit_linesZ
pair_specsZn_pairsr%   rC   rJ   
componentspairjnameidxZperson_folder	filenamesrK   pairsr   rH   r)   r)   r*   _fetch_lfw_pairsm  sB    	

r   train)subsetr   r#   r@   rE   rD   r$   c                 C   s   t |||d\}}td| | t|ddd}	|	t}
dddd	}| |krhtd
| tt|	 f t
|||  }|
|||||d\}}}td}t|t|d||||dS )a  Load the Labeled Faces in the Wild (LFW) pairs dataset (classification).

    Download it if necessary.

    =================   =======================
    Classes                                   2
    Samples total                         13233
    Dimensionality                         5828
    Features            real, between 0 and 255
    =================   =======================

    In the official `README.txt`_ this task is described as the
    "Restricted" task.  As I am not sure as to implement the
    "Unrestricted" variant correctly, I left it as unsupported for now.

      .. _`README.txt`: http://vis-www.cs.umass.edu/lfw/README.txt

    The original images are 250 x 250 pixels, but the default slice and resize
    arguments reduce them to 62 x 47.

    Read more in the :ref:`User Guide <labeled_faces_in_the_wild_dataset>`.

    Parameters
    ----------
    subset : {'train', 'test', '10_folds'}, default='train'
        Select the dataset to load: 'train' for the development training
        set, 'test' for the development test set, and '10_folds' for the
        official evaluation set that is meant to be used with a 10-folds
        cross validation.

    data_home : str, default=None
        Specify another download and cache folder for the datasets. By
        default all scikit-learn data is stored in '~/scikit_learn_data'
        subfolders.

    funneled : bool, default=True
        Download and use the funneled variant of the dataset.

    resize : float, default=0.5
        Ratio used to resize the each face picture.

    color : bool, default=False
        Keep the 3 RGB channels instead of averaging them to a single
        gray level channel. If color is True the shape of the data has
        one more dimension than the shape with color = False.

    slice_ : tuple of slice, default=(slice(70, 195), slice(78, 172))
        Provide a custom 2D slice (height, width) to extract the
        'interesting' part of the jpeg files and avoid use statistical
        correlation from the background.

    download_if_missing : bool, default=True
        If False, raise a IOError if the data is not locally available
        instead of trying to download the data from the source site.

    Returns
    -------
    data : :class:`~sklearn.utils.Bunch`
        Dictionary-like object, with the following attributes.

        data : ndarray of shape (2200, 5828). Shape depends on ``subset``.
            Each row corresponds to 2 ravel'd face images
            of original size 62 x 47 pixels.
            Changing the ``slice_``, ``resize`` or ``subset`` parameters
            will change the shape of the output.
        pairs : ndarray of shape (2200, 2, 62, 47). Shape depends on ``subset``
            Each row has 2 face images corresponding
            to same or different person from the dataset
            containing 5749 people. Changing the ``slice_``,
            ``resize`` or ``subset`` parameters will change the shape of the
            output.
        target : numpy array of shape (2200,). Shape depends on ``subset``.
            Labels associated to each pair of images.
            The two label values being different persons or the same person.
        target_names : numpy array of shape (2,)
            Explains the target values of the target array.
            0 corresponds to "Different person", 1 corresponds to "same person".
        DESCR : str
            Description of the Labeled Faces in the Wild (LFW) dataset.
    re   zLoading %s LFW pairs from %srf   r   rg   r   r   r   )r   testZ10_foldsz+subset='%s' is invalid: should be one of %r)r@   rE   rD   rl   rk   )rm   r   r%   r]   rn   )r+   r   r    r   ro   r   rW   r~   rT   keysr   r   r   rp   r<   )r   r   r#   r@   rE   rD   r$   r   r&   rq   rr   Zlabel_filenamesr   r   r%   r]   rt   r)   r)   r*   fetch_lfw_pairs  sD    Z  

    r   )NTT)NFNr   )NFN)"__doc__osr   r   r   os.pathr   r   r   loggingZnumpyr=   Zjoblibr   _baser
   r   r   r   utilsr   	getLogger__name__r   r   r   r   r+   rM   r_   r4   ru   r   r   r)   r)   r)   r*   <module>   s~   


+K       
-      
6