U
    db                     @   sV   d dl mZmZmZ d dlZd dlm  mZ d dlm	Z	 ej
jG dd deZdS )    )ListOptionalDictN)Tensorc                   @   sZ   e Zd Zdee eeeeeeeed	ddZeee ddd	Z	eee  d
ddZ
dS )_FunctionalSGD{Gz?        F)	paramslrmomentum	dampeningweight_decaynesterovmaximizeforeach_allow_empty_param_listc
           
      C   sl   ||||d| _ || _|| _|| _tjttjtt	tjf f i | _
t|dkr^|	s^tdd|i| _d S )N)r
   r   r   r   r   z%optimizer got an empty parameter listr	   )defaultsr   r   r   torchjitZannotater   r   strstatelen
ValueErrorparam_group)
selfr	   r
   r   r   r   r   r   r   r    r   J/tmp/pip-unpacked-wheel-ua33x9lu/torch/distributed/optim/functional_sgd.py__init__   s    $z_FunctionalSGD.__init__)paramgradc                 C   s   | j d }| j d }| j d }| j d }|g}g }g }	d}
|dk	r|	| |jrVd}
|| jkrji | j|< | j| }d|kr|d n||d  t , tj||	|||||| j| j	|
| j
d	 W 5 Q R X | j| }|d
 }|dk	r||d< dS )z` Similar to self.step, but operates on a single parameter and
            its gradient.
        r   r   r   r
   FNTmomentum_bufferr   r   r
   r   r   r   has_sparse_gradr   r   )r   append	is_sparser   r   no_gradFsgdr   r   r   )r   r   r   r   r   r   r
   r	   momentum_buffer_listgradsr"   r   r    r   r   r   
step_param0   sH    









z_FunctionalSGD.step_param)	gradientsc                 C   sh  | j d }g }g }g }| jd }| jd }| jd }| jd }	t|t|krttddt| d d	t|  d
}
t||D ]n\}}|d k	r|| || |jrd}
|| jkri | j|< | j| }d|kr|d  q||d  qt	 , t
j|||||||	| j| j|
| jd W 5 Q R X t|D ].\}}| j| }|| }|d k	r4||d< q4d S )Nr	   r
   r   r   r   zEthe gradients passed in does not equal to the size of the parameters!zParams length: z. zGradients length: FTr    r!   )r   r   r   r   zipr#   r$   r   r   r%   r&   r'   r   r   r   	enumerate)r   r+   r	   Zparams_with_gradr)   r(   r
   r   r   r   r"   r   Zgradientr   ipr    r   r   r   step_   s^    












z_FunctionalSGD.stepN)r   r   r   r   FFFF)__name__
__module____qualname__r   r   floatboolr   r   r*   r0   r   r   r   r   r      s*           /r   )typingr   r   r   r   Ztorch.optim._functionalZoptimZ_functionalr&   r   r   scriptobjectr   r   r   r   r   <module>   s
   