U
    d                     @   sZ   d dl mZmZmZmZ d dlZd dlm  mZ	 d dlm
Z
 ejjG dd deZdS )    )ListDictOptionalTupleN)Tensorc                
   @   sL   e Zd Zdee eeeef eeeeedddZee	e  d	d
dZ
dS )_FunctionalAdamaxMbP?g?g+?:0yE>        F)paramslrbetasepsweight_decayforeachmaximize_allow_empty_param_listc	           	      C   s
  d|kst d|d|ks,t d|d|d   krDdk sXn t d|d d|d   krpdk sn t d|d d|kst d	||||d |d |d
| _|| _|| _tjttj	tt
tj	f f i | _t|dkr|st dd|i| _d S )Nr   zInvalid learning rate: {}zInvalid epsilon value: {}r   g      ?z%Invalid beta parameter at index 0: {}   z%Invalid beta parameter at index 1: {}zInvalid weight_decay value: {})r   r   beta1beta2r   z%optimizer got an empty parameter listr   )
ValueErrorformatdefaultsr   r   torchjitZannotater   r   strstatelenparam_group)	selfr   r   r   r   r   r   r   r    r!   M/tmp/pip-unpacked-wheel-ua33x9lu/torch/distributed/optim/functional_adamax.py__init__   s,    $z_FunctionalAdamax.__init__)	gradientsc                 C   sn  | j d }g }g }g }g }g }t|t|krTtddt| d dt|  t| j d |D ]\}}	|	d k	rd|| ||	 || jkri | j|< | j| }
td|
d< tj|tj	d|
d	< tj|tj	d|
d
< | j| }
||
d	  ||
d
  ||
d  qdt
 J tj|||||| jd | jd | jd | jd | jd | j| jd W 5 Q R X d S )Nr   zEthe gradients passed in does not equal to the size of the parameters!zParams length: z. zGradients length: r   step)Zmemory_formatZexp_avgZexp_infr   r   r   r   r   )r   r   r   r   r   r   r   )r   r   r   zipappendr   r   ZtensorZ
zeros_likeZpreserve_formatZno_gradFZadamaxr   r   r   )r    r$   r   Zparams_with_gradZgradsZexp_avgsZexp_infsZstate_stepsparamZgradientr   r!   r!   r"   r%   :   sT    







z_FunctionalAdamax.stepN)r   r	   r
   r   FFF)__name__
__module____qualname__r   r   floatr   boolr#   r   r%   r!   r!   r!   r"   r      s$          
(r   )typingr   r   r   r   r   Ztorch.optim._functionalZoptimZ_functionalr(   r   r   scriptobjectr   r!   r!   r!   r"   <module>   s
   