U
    d$                     @   s   d dl Z d dl mZ ddlmZ d dlmZmZ G dd deZdee ee ee ee ee	e	e	e	d	d	d
Z
ee ee ee ee e	e	e	e	dddZee ee ee ee e	e	e	e	dddZdS )    N)Tensor   )	Optimizer)ListOptionalc                       sJ   e Zd ZdZdee d fddZ fd	d
Ze	 dddZ
  ZS )Rpropa  Implements the resilient backpropagation algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \theta_0 \in \mathbf{R}^d \text{ (params)},f(\theta)
                \text{ (objective)},                                                             \\
            &\hspace{13mm}      \eta_{+/-} \text{ (etaplus, etaminus)}, \Gamma_{max/min}
                \text{ (step sizes)}                                                             \\
            &\textbf{initialize} :   g^0_{prev} \leftarrow 0,
                \: \eta_0 \leftarrow \text{lr (learning rate)}                                   \\
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm} \textbf{for} \text{  } i = 0, 1, \ldots, d-1 \: \mathbf{do}            \\
            &\hspace{10mm}  \textbf{if} \:   g^i_{prev} g^i_t  > 0                               \\
            &\hspace{15mm}  \eta^i_t \leftarrow \mathrm{min}(\eta^i_{t-1} \eta_{+},
                \Gamma_{max})                                                                    \\
            &\hspace{10mm}  \textbf{else if}  \:  g^i_{prev} g^i_t < 0                           \\
            &\hspace{15mm}  \eta^i_t \leftarrow \mathrm{max}(\eta^i_{t-1} \eta_{-},
                \Gamma_{min})                                                                    \\
            &\hspace{15mm}  g^i_t \leftarrow 0                                                   \\
            &\hspace{10mm}  \textbf{else}  \:                                                    \\
            &\hspace{15mm}  \eta^i_t \leftarrow \eta^i_{t-1}                                     \\
            &\hspace{5mm}\theta_t \leftarrow \theta_{t-1}- \eta_t \mathrm{sign}(g_t)             \\
            &\hspace{5mm}g_{prev} \leftarrow  g_t                                                \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to the paper
    `A Direct Adaptive Method for Faster Backpropagation Learning: The RPROP Algorithm
    <http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.21.1417>`_.

    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-2)
        etas (Tuple[float, float], optional): pair of (etaminus, etaplis), that
            are multiplicative increase and decrease factors
            (default: (0.5, 1.2))
        step_sizes (Tuple[float, float], optional): a pair of minimal and
            maximal allowed step sizes (default: (1e-6, 50))
        foreach (bool, optional): whether foreach implementation of optimizer
            is used (default: None)
    {Gz?g      ?g333333?gư>2   N)foreachc                    s|   d|kst d|d|d   k r<d  k r<|d k sVn t d|d |d t||||d}tt| || d S )Ng        zInvalid learning rate: {}r   g      ?r   zInvalid eta values: {}, {})lretas
step_sizesr   )
ValueErrorformatdictsuperr   __init__)selfparamsr   r   r   r   defaults	__class__ 5/tmp/pip-unpacked-wheel-ua33x9lu/torch/optim/rprop.pyr   8   s    (zRprop.__init__c                    s(   t  | | jD ]}|dd  qd S )Nr   )r   __setstate__param_groups
setdefault)r   stategroupr   r   r   r   B   s    
zRprop.__setstate__c                 C   s>  d}|dk	r&t   | }W 5 Q R X | jD ]
}g }g }g }g }|d \}}	|d \}
}|d }|d D ]}|jdkrzqj|| |j}|jrtd|| | j| }t|dkrd|d< t j	|t j
d	|d
< | ||d |d< ||d
  ||d  |d  d7  < qjt|||||
|||	|d	 q,|S )zPerforms a single optimization step.

        Args:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   z'Rprop does not support sparse gradientsr   stepZmemory_formatprevr   	step_sizer   )step_size_minstep_size_maxetaminusetaplusr   )torchZenable_gradr   gradappendZ	is_sparseRuntimeErrorr   lenZ
zeros_likepreserve_formatnewZ
resize_as_Zfill_rprop)r   closureZlossr    r   gradsprevsr   r'   r(   r%   r&   r   pr*   r   r   r   r   r!   G   sN    





z
Rprop.step)r   r	   r
   N)N)__name__
__module____qualname____doc__r   boolr   r   r)   Zno_gradr!   __classcell__r   r   r   r   r      s   0  
r   )	r   r2   r3   r   r   r%   r&   r'   r(   c          
   
   C   sV   |dkrd}|r"t j r"td|r6t j s6t}	nt}	|	| |||||||d dS )zpFunctional API that performs rprop algorithm computation.

    See :class:`~torch.optim.Rprop` for details.
    NFz6torch.jit.script not supported with foreach optimizers)r%   r&   r'   r(   )r)   ZjitZis_scriptingr,   _multi_tensor_rprop_single_tensor_rprop)
r   r2   r3   r   r   r%   r&   r'   r(   funcr   r   r   r0      s     r0   )r   r2   r3   r   r%   r&   r'   r(   c                C   s   t | D ]\}}	|| }
|| }|| }|
| }|||d< |||d< d||d< |||| |
jt	j
d}
d|
||< |	j|
 |dd ||
 qd S )Nr   r   r"   value)	enumeratemulsigngtlteqZmul_clamp_cloner)   r.   Zaddcmul_copy_)r   r2   r3   r   r%   r&   r'   r(   iparamr*   r#   r$   rC   r   r   r   r<      s    
r<   c                C   s  t | dkrd S t||}dd |D }|D ].}	||	|	d< ||	|	d< d|	|	d< q.t|| |D ]}
|
|| qntt |D ]0}|| j	tj
d||< d|| || |< qdd |D }tj| ||dd tt |D ]}|| ||  qd S )	Nr   c                 S   s   g | ]}|  qS r   rC   ).0sr   r   r   
<listcomp>   s     z'_multi_tensor_rprop.<locals>.<listcomp>r   r"   c                 S   s   g | ]}|  qS r   rL   )rM   r*   r   r   r   rO      s     r>   r?   )r-   r)   Z_foreach_mulrD   rE   rF   Z_foreach_mul_rG   rangerH   r.   Z_foreach_addcmul_rI   )r   r2   r3   r   r%   r&   r'   r(   ZsignsrC   r$   rJ   Z
grad_signsr   r   r   r;      s$    
r;   )N)r)   r   Z	optimizerr   typingr   r   r   r9   floatr0   r<   r;   r   r   r   r   <module>   sB    '"