U
    dg                    @   sl  d Z ddlZddlZddlZddlZddlmZmZmZm	Z	 ddl
Z
ddlm  mZ ddlZ
ddlZ
ddl
mZ ddlmZ ddlmZ ddlmZ dd	 Zd
d Zdd Zdd Zdd ZdfddZdgddZdhddZdd Zdd Zedddddidd Z eddd!d"d# Z!d$d% Z"d&d' Z#d(d) Z$d*d+ Z%d,d- Z&d.d/ Z'edd0d1d2 Z(edd0d3d4 Z)d5d6 Z*d7d8 Z+d9d: Z,d;d< Z-edddd=d=d>d? Z.d@dA Z/dBdC Z0dDdE Z1dFdG Z2dHdI Z3dJdK Z4dLdM Z5dNdO Z6dPdQ Z7dRdS Z8ej9dTdUddVdWdX Z:dYdZ Z;d[d\ Z<d]d^ Z=djd_d`Z>dadb Z?dkdcddZ@e@dedfZAe@dgdhZBe@didjdkdlZCedd0dmdndo ZDdpdq ZEdrds ZFdtd= ZGdudv ZHdwdx ZIeddd0dyddzd{ ZJedddd0d0d0dd0d0	d|d} ZKdld~dZLedd0d0dd ZMedddd ZNdd ZOdd ZPedd0d0d0dmddZQedddd0dnddZRdoddZSeddd0d0dpddZTdqddZUedd0d0drddZVedd0ddd ZWdd ZXdsddZYdd ZZdd Z[dd Z\dd Z]e9dTdd Z^e9dTdd Z_dd Z`dd Zadd Zbedd=d=dd ZcdtddZdedd0dd Zeedd0dmduddZfdd Zgdd Zhdd Zieide
jjjkjljmddkdZneide
jjjkjljoddkdZpeide
jjjkjljqddkdZreide
jjjkjljmddTdZseide
jjjkjljoddTdZteide
jjjkjljqddTdZuddƄ Zvevde
jjjkjljmZwevde
jjjkjljoZxevde
jjjkjljqZydvdd˄Zzezdde
jjjkjljmZ{ezdde
jjjkjljoZ|ezdde
jjjkjljqZ}ezdde
jjjkjljmesZ~ezdde
jjjkjljoetZezdde
jjjkjljqeuZddՄ Zddׄ Zddل Zddۄ Zdd݄ Zdd߄ ZeZeZeZeZeZeZdd Zdd ZedddZedddZedddZedddZedddZedddZdd Zdd Zdd Zdd Zdd Zdd Zdd Zedd Zdd Zd d Zdd Zdd Zedd Zedd	 Zd
d Zdd Zdd Zeddd Zeddd Zeddd Zdd Zdd Zedddd0dwddZedd0dmdxddZeddddddd0dd0d0d0d0d0dydd Zeddddddd0d!d" Zeddddddd0d#d$ Zeddddddd0d%d& Zeddddddd0dd'd( Zeddddddd0dd)d* Zeddddddd0dd+d, Zedddddd0ddd0	d-d. Zedddddd0d/d0 Zedddddd0ddd0	d1d2 Zedd0d0d0d3d4 Zedd=d=d=d5d6 Zd7d8 Zedd0dd9d: Zd;d< Zd=d> Zd?d@ ZedddydydzdAdBZdCdD Zeddd0ddEdF ZdGdH ZdIdJ ZdKdL ZÐdMdN ZĐdOdP ZŐdQdR ZƐdSdT ZǐdUdV ZedddWdX ZedddYdZ Zʐd{d[d\Zːd]d^ Z̐d|d_d`Z͐dadb Zeddd0dcdd Zeddd0dedf Zeddd0dgdh Zѐdidj Zeddd0dkdl ZӐdmdn ZeԐdoZeԐdpZeԐdqZeZeZeZeZedd=dd0drds Zedddd0dtdu Zedd0d0dvdw Zedd0d0d0dxdy Zej D ]6\ZZdzeZedd0eejee e< qedd0ddddd}d{d|Zedd0ddddd~d}d~ZdddZdd ZdddZdddZedd0ddddddZedd0dddddddZdddZedd0ddddddZedd0dddddddZdddZdddZdddZdddZdd Zdd Zeddddd Zeddd Zej9dTdUddVeddd Zeddd Zedddd Zedddd Zdd Z edd0dd Zedd0d0dmdddZdd Zedd0d0d0d0dmdddZdd Zdd ZdddZedd0dd Zedd0dd Z	dddZ
edddd0d0dd0d0d0	dd Zeddddd0d0dd0d0	dd ZdÐdĄ ZdŐdƄ ZdǐdȄ ZedɃZedʃZed˃Zedd0d̐d̈́ Zdΐdτ Zedd0dАdф Zeddd0dҐdӄ Zeddd0d=ddԐdՄ Zd֐dׄ Zdؐdل ZddڐdۄZddܐd݄Zedddd0dmdސd߄ ZdddZeddd Zeddd Ze9dTdkdkedd0d0dd Z eddd Z!dddZ"eddd Z#dd Z$dd Z%edd0d0d0dd Z&dd Z'dd Z(edd0dddd Z)edd0dddd Z*dd Z+dd Z,d d Z-edd Z.dd Z/edd0dddddZ0eddd0d0dd	 Z1d
d Z2dd Z3dd Z4dd Z5eddd0dd Z6dd Z7dd Z8dd Z9dd Z:dd Z;edddd0ddd Z<edddd0dd d! Z=edddd0dd"d# Z>eddd0dd%d&Z?eddd0dd'd(Z@edd0dyddd)d*ZAd+d, ZBedd!deeC d-d.d/ZDd0d1 ZEedd!de
jjFeCd2d3d4ZGedd0dddd0d5d6 ZHeddd0d7d8 ZId9d: ZJd;d< ZKd=d> ZLd?d@ ZMdAdB ZNdCdD ZOeddd0dydEdF ZPedddd0ddGdHZQdIdJ ZRdKdL ZSdMd ZTeddyd0ddddddNdOZUdPdQ ZVdRdS ZWedddTdU ZXddVdWZYeddddXdY ZZeddd0ddZd[Z[dd^d_Z\d`da Z]G dbdc dcZ^G ddde deZ_dS (  zhThis file exports ONNX ops for opset 9.

Opset 9 is supported by ONNX release 1.4.1
release on 01/23/19
    N)ListOptionalTupleUnion)_C)_patch_torch)symbolic_helper)GLOBALSc                 C   s   |  d}|tj  |S )Nprim::Constant)opsetTyper   OptionalTypeZofTensor)gn r   >/tmp/pip-unpacked-wheel-ua33x9lu/torch/onnx/symbolic_opset9.pyunused9   s    
r   c                 C   s   |  d|S NShaper   r   inputr   r   r   _shape_as_tensor?   s    r   c                 C   s*   t |tr| jd|ddi}t| ||S )NConcataxis_ir   )r   )
isinstancelistr   reshape)r   r   shaper   r   r   _reshape_from_tensorC   s    
r   c                 C   s   t | ||S N)r   _reshape_helperr   selfr   r   r   r   r   I   s    r   c                 C   s   |  d|}t| ||S r   r   r   r   r#   otherr   r   r   r   
reshape_asM   s    r'   c                 C   sV   t |r$t |r$t ddddS |rHt t |dkrHt ddS | d||S )NAdd	      z)Add between list of tensors not supported   add
alpha != 1)r   	_is_value_is_tensor_list _onnx_opset_unsupported_detailed_scalar_maybe_get_scalar_unimplementedr   r   r#   r&   alphar   r   r   r,   R   s       r,   c                 C   s2   |r$t t |dkr$t ddS | d||S )Nr+   subr-   Sub)r   r1   r2   r3   r   r4   r   r   r   r6   ^   s    r6   c                 C   s   t | |||dS )N)r5   )r6   r4   r   r   r   rsube   s    r8   c                 C   s   |  d||S NMulr   r   r#   r&   r   r   r   muli   s    r<   c                 G   s.   t |dkrt| ||S t| ||f| S d S Nr   )lentrue_divide_div_rounding_mode)r   r#   r&   argsr   r   r   divm   s    rB   vf      ?c              	   C   s2   | j dt|gd}t| |t| t| |||S )NConstantZvalue_t)r   torchtensorr,   r<   )r   r#   Ztensor1Ztensor2valueZ
value_tensr   r   r   addcmult   s    rK   sc                 C   sP   |d krt | ||S |dkr(t| ||S |dkr<t| ||S td| dd S )NfloortrunczUnsupported rounding mode: "z$". Expected None, "floor" or "trunc")r?   _floor_divide_trunc_divideRuntimeError)r   r#   r&   Zrounding_moder   r   r   r@   z   s    
r@   c                 C   s   |  d||}| j d|tjd d}|  }|d k	rt|st|  d k	rtt|rt| j d|tjd d}q| j d|tj| d}n| j d|tjd d}|S )NDivCastLongZto_iFloat)r   r   cast_pytorch_to_onnxtype
scalarType_is_fp)r   r#   r&   outscalar_typer   r   r   rP      s$      rP   c                 C   s   t |st |r,t| ||}| d|S | d||}| jdtjdtjdd}| dt | ||t | ||}| d|| d	||}| d
|| d| d||}| jdtjdtjdd}	| d	||	}
| d||
S d S )NFloorrR   rF   r   dtyperG   Xorr7   r:   AndNotEqualr+   )r   rZ   r?   r   rH   rI   int64Z
_lt_helper)r   r#   r&   r[   rB   ZzeronegativemodZ
fixup_maskonefixupr   r   r   rO      s     rO   c                 C   s   t | ||S r    )rP   r;   r   r   r   floor_divide   s    ri   c                 C   s   t | ||S r    )ri   r;   r   r   r   floordiv   s    rj   c                 C   s   t |st |r"| d||S t }t jd }|tjksL|tjksLtt tjkrdt jd }| jd||d}| jd||d}| d||S )a  Division where both inputs are cast to floating types

    If both inputs are floating, performs div as usual
    If only one input is a floating type, the other input is cast to its type
    If neither input is a floating type, both inputs are cast to the default scalar type
    rR   rV   DoublerS   rU   )	r   rZ   r   rH   get_default_dtyperW   floatdoubleAssertionError)r   r#   r&   r\   Zonnx_scalar_typer   r   r   r?      s    

r?   c                 C   s,   t |s | jd|t jd d}| d|S )NrS   rV   rU   Z
Reciprocal)r   rZ   r   rW   r   r#   r   r   r   
reciprocal   s    
rq   ic                 C   s   t |}| jd|d|iS )Nr   r   )r   r   _unpack_listr   )r   tensor_listdimtensorsr   r   r   cat   s    
rx   c                    s.    fddt |D }jd|d iS )Nc                    s   g | ]}t | gqS r   r   _unsqueeze_helper.0trv   r   r   r   
<listcomp>   s   zstack.<locals>.<listcomp>r   r   )r   rs   )r   ru   rv   Z
unsqueezedr   r~   r   stack   s    r   c                 C   s   |S r    r   rp   r   r   r   _list   s    r   c                 C   s,   | j dtdgd}| j d|||dddS )NrF   r+   rG   Gemm        rE   Zbeta_falpha_f)r   rH   rI   )r   r#   r&   Cr   r   r   mm   s    r   c                 C   s   |  d||S NMatMulr   r;   r   r   r   bmm  s    r   c                 C   s   |  d||S r   r   r;   r   r   r   matmul	  s    r   r}   c              	   C   s\  d }t |}t |}t |}	|d k	r0|}n|d k	r>|}n|	d k	rJ|	}t |}
t |}dd }|d k	r:||
ds||dr:t jt j| }t j| }| d||}|}t |}t |}|dkr| jdt	j
||dd}| d	||}|dkr,| jdt	j
t ||dd}| d	||}| d
||S | jd|||t |t |dS )Nc                 S   s   | d k	o| |kS r    r   )rC   ur   r   r   isNotNoneAnd  s    zaddmm.<locals>.isNotNoneAnd   r   r+   rF   r^   rG   r:   r(   r   r   )r   _try_get_scalar_type_get_tensor_rankscalar_type_to_onnxindexrW   scalar_type_to_pytorch_typer   r1   rH   rI   )r   r#   Zmat1Zmat2betar5   r_   
self_dtypeZ
mat1_dtypeZ
mat2_dtypeZ	mat1_rankZ	mat2_rankr   Zres1Zres2r   r   r   addmm  sP    




 



r   c                 C   s   |  d|S )NZNegr   rp   r   r   r   negB  s    r   c                 C   s   |  d|S NSqrtr   rp   r   r   r   sqrtF  s    r   c              	   C   s$   |  dt| td|t| |S )NrR   r+   )r   r   _if_scalar_type_asrH   onesr   rp   r   r   r   rsqrtJ  s
      r   c                 C   s   |  d|S )NTanhr   rp   r   r   r   tanhP  s    r   c                 C   s   |  d|S )NZSinr   rp   r   r   r   sinT  s    r   c                 C   s   |  d|S )NZCosr   rp   r   r   r   cosX  s    r   c                 C   s   |  d|S )NZTanr   rp   r   r   r   tan\  s    r   c                 C   s   |  d|S )NZAsinr   rp   r   r   r   asin`  s    r   c                 C   s   |  d|S )NZAcosr   rp   r   r   r   acosd  s    r   c                 C   s   |  d|S )NZAtanr   rp   r   r   r   atanh  s    r   Tg      p?)scaleZ
zero_pointc                 C   s   |  d|S )NSigmoidr   rp   r   r   r   sigmoidm  s    r   c                 C   s   |  d|S )NZSignr   rp   r   r   r   signr  s    r   c                 C   sP   t |t |kstt |dkr<|d dkr<|d dkr<|S | jd||||dS )Nr+   r       ZSlice)axes_iZstarts_iZends_i)r>   ro   r   )r   r   axesstartsendsr   r   r   _slicev  s    $r   c                 C   s6   |   }|d k	r2t|s2|dks2t| |d}|S )NrT   F)rX   rY   r   rZ   
_cast_Longr   r#   r_   r   r   r   _maybe_cast_reduce_op_input}  s
    r   c                    s   d fdd	}|S )Nc                    sn   t | |}|d kr t| |S  r(dnd}t||dt|dd }} rR|n|g}| j|||dS d S )Nisrr   rv   keepdimr   
keepdims_i)r   r   Z_handle_reduce_dim_none
_get_constr   )r   r#   rv   r   descZdim_listallow_multi_dim_supportonnx_op_namer   r   symbolic  s    
  z%_reduce_op_symbolic.<locals>.symbolic)NNr   )r   r   r   r   r   r   _reduce_op_symbolic  s    r   c                    s   t   fdd}|S )Nc                    sX    | f| }d }|D ],}|j }t|t|kr|| f|   S qtd jd S )NzUnknown aten::{} signature)Z_arg_descriptorsr>   NotImplementedErrorformat__name__)r   rA   Z	overloadsZlast_exceptionoverloadZarg_descriptorsfnr   r   wrapper  s    z&overload_by_arg_count.<locals>.wrapper)	functoolswraps)r   r   r   r   r   overload_by_arg_count  s    	r   c                    s$   t |  dt fdd}|S )Nr   c                    sL   t ddfdd} r"dnd}t d|ddfdd}||fS )	NrC   nonec                    s\   |   dkr6t|dd}| jd|tj| d}n|   dkrRt dS | |S Nonnx::Constantrr   r_   rS   rU   r
   nodekindr   r   r   r   r3   r   namer   r   r   reduce_nodim  s      z8_reduce_with_dtype.<locals>.reduce.<locals>.reduce_nodimr   rr   c                    s`   |   dkr6t|dd}| jd|tj| d}n|   dkrRt dS | |||S r   r   )r   r#   rv   r   r_   r   r   r   
reduce_dim  s      z6_reduce_with_dtype.<locals>.reduce.<locals>.reduce_dimr   
parse_args)r   rA   kwargsr   Zdim_descr   r   r   r   r   r   reduce  s    


z"_reduce_with_dtype.<locals>.reduce)r   r   )Zonnx_opr   r   r   r   r   r   _reduce_with_dtype  s     r   Z	ReduceSumsum
ReduceMeanmean
ReduceProdprodFr   r   c                 C   sF   t  r4|  dkr$t tdS | jd||dS t ddd d S )Nr
   r_   cumsumdim_ir)   r*   )r   is_caffe2_aten_fallbackr   r   r3   r   at_onnx_opset_unsupported)r   r   rv   r_   r   r   r   r     s
    r   c                 C   s8   t  r*t |st ddS | d|S t dS d S )N_sample_dirichlet#We are not able to export generatorr   r   _is_noner3   r   _onnx_unsupportedr   r#   	generatorr   r   r   r     s    
 r   c                 C   s8   t  r*t |st ddS | d|S t dS d S )N_standard_gammar   r   r   r   r   r   r     s    
 r   c                 C   s   | j d|ddS )N	Transpose)r+   r   Zperm_ir   rp   r   r   r   r}     s    c              	   C   s   t |d}t |s,| jdt|d}n2t |r^t | t| |d| jdt	dgd}t j
j}t| ||}t| || jdt	dd}t| | d||||}| d||S )Nr   rF   rG   r   rc   Expand)r   _maybe_get_constr.   r   rH   
LongTensor_is_packed_listr!   r   rI   
ScalarTypeINT64	ones_liker<   where)r   r#   sizeZimplicitr_   r   Zneg_onesr   r   r   expand  s    

 
 r   c                 C   s   t |d}t|tjr|j}|tj}g }t|	 D ]F}t
|||||r:|| | jd|||d}q:| d|}| d||S )Nr}   rF   rG   r   r   )r   r   r   rH   Tensorr_   torn   rangerv   equalr   	unsqueeze	expand_asappendr   )r   r#   r&   Zself_t	orig_typedimsdr   r   r   r   r     s    
r   bc                 C   s8   |rt jrtd|dkr*t jr*td | d||S )NzUnsupported: ONNX export of embedding with scale_grad_by_freq=True for training mode. ONNX does not support scaling the gradients.r   zWarning: ONNX export of embedding with padding_idx >= 0 for training mode. ONNX does not support not updating the embedding vector at padding_idx during training.Gather)r	   Ztraining_moderQ   warningswarnr   )r   weightindicespadding_idxscale_grad_by_freqsparser   r   r   	embedding  s    
r  c
           
      C   sH   t |st dS t  r:| jd|||d|||||	d
S t dS d S )Nz&embedding_bag  with per_sample_weightsembedding_bag   )outputsZscale_grad_by_freq_iZmode_iZsparse_iZinclude_last_offset_iZpadding_idx_i)r   r   r   r   r   )
r   Zembedding_matrixr  offsetsr
  moder  Zper_sample_weightsZinclude_last_offsetr	  r   r   r   r  '  s$    
r  c                 C   sh   |d kr|  d|S t|ddk rZt|}|d k	rZt|d| }| j dt|d}t| ||S )Nr   rr   r   rF   rG   )r   r   r   r   rH   rI   Z_size_helperr   r#   rv   rankr   r   r   r   I  s    
r   c                 C   s|   ||kr|S t |}|d k	rTtt|}|| ||  ||< ||< | jd||dS t  rp| jd|d||dS tdd S )Nr   r   	transposeint)overload_nameZdim0_iZdim1_izAUnsupported: ONNX export of transpose for tensor of unknown rank.)r   r   r   r   r   r   r   rQ   )r   r#   Zdim0Zdim1r  r   r   r   r   r  T  s$    
    r  r   c                 C   s*   |t tdt|kr|S | jd||dS )Nr   r   r   )r   r   r>   r   )r   r#   r  r   r   r   permutel  s    r  c                 C   s   t | ||S r    )r   )r   r#   r   r   r   r   views  s    r  c                 C   s   |  d|}t| ||S r   r$   r%   r   r   r   view_asw  s    r  c           	      C   s   |d krt ddddS t ||}|d kr8t ddS || d | }|g||  }|| }|rl|| | jd||||dS )	Nunsafe_chunkr)   r*   'Dynamic number of outputs not supportedunknown dimension sizer+   SplitZsplit_ir   r  )r   r0   _get_tensor_dim_sizer3   r   r   )	r   r#   chunksrv   _outputsr   
split_sizesplitsleftoverr   r   r   r  |  s        
r  c           
      C   s   t ||st ddddS | d }| dkrDt| ||||S t |dd}t |dd	}t ||}|d kr|d k	r|| }nt dddd
S |g||  }|| }	|	r||	 | j	d||||dS )Nsplitr)   r*   r  rJ   r   rr   r"  rv   z$Unknown dimension size not supportedr  r  )
r   _is_split_staticr0   r   rv   split_with_sizesr   r  r   r   )
r   r#   split_size_or_sizesrv   r!  Z	split_valr"  r   r#  r$  r   r   r   r%    s6       
   
r%  c                 C   s   t | ||||S r    )r%  )r   r#   r(  rv   r!  r   r   r   unsafe_split  s    r)  c                 C   s0   t ||st ddddS | jd||||dS )Nr'  r)   r*   r  r  r  )r   r&  r0   r   r   r#   Zsplit_sizesrv   r!  r   r   r   r'    s       r'  c                 C   s   t | ||||S r    )r'  r*  r   r   r   unsafe_split_with_sizes  s    r+  c                    s\   |d krt ddddS jd|dg|  |d}|dkr@|gn|} fdd	|D }|S )
Nunbindr)   r*   r  r  r+   r  c                    s   g | ]}t | gqS r   )r   _squeeze_helper)r|   r[   r~   r   r   r     s    zunbind.<locals>.<listcomp>)r   r0   r   )r   r#   rv   r!  r  Zsqueezed_outputsr   r~   r   r,    s       r,  c                 C   sr   t |}t |s\|dk r\|dkr*d}n|d }t j| ||g|g|gd}t | ||gS | jd|||dS d S )Nr   r   r   r+   r   r   r   r  r   )r   r2   r.   _slice_helperr-  r   )r   r#   rv   r   Z	end_indexZ
slice_noder   r   r   select  s    
    r1  c                 C   s   |  d||S r9   r   rp   r   r   r   square  s    r2  c                 C   sH  |d kr|  d|S t|dd}|dk rt|}|d k	rxtdt| d d d t||  d	 d
  ||7 }ntddS t||}|d krtdt| d d t| d d d d  tj	| ||gdS |dkrtdt| d d t| d d d d  |S tdt| d d  tj	| ||gdS )NZSqueezerr   rv   r   z'ONNX export squeeze with negative axis - might cause the onnx model to be incorrect. (Negative axis is not supported in ONNX. Axis is converted to & based on input shape at export time. CPassing an tensor of different rank in execution will be incorrect.squeeze%negative axis with unknown input rankz5This model contains a squeeze operation on dimension z on an input z7with unknown shape. Note that if the size of dimension z of the input zVis not 1, the ONNX model will return an error. Opset version 11 supports squeezing on zMnon-singleton dimensions, it is recommended to export this model using opset zversion 11 or higher.r   r+   z. The size of z%this dimension in the given input is z. The model will zWbe exported without the squeeze node. If the model is intended to be used with dynamic z-input shapes, please use opset version 11 to zexport the model.z. If the model is z_intended to be used with dynamic input shapes, please use opset version 11 to export the model.)
r   r   r   r   r  r  strr3   r  r-  )r   r#   rv   Zsqueeze_dimr  dim_sizer   r   r   r8    s    



 
r8  c              	   C   s   t |}|d k	rT|dkr8t | |ttd|d }n|dkrTt | |dg}d}t |}|d k	r|d k	r||kstd||| d||S )Nr   r+   r   z0rank(x) should be >= rank(slope) but got {} < {}PRelu)r   r   rz   r   r   ro   r   r   )r   r#   r  Z	self_rankZweight_rankr   r   r   prelu  s*    
  
 r>  c                 C   s   |  d||  d|S )Nr:   r   r   r   r   r   r   silu3  s    r?  c                 C   s   |  d||  d|  d|S )Nr:   r   Softplusr   r   r   r   r   mish7  s    rA  c              	   O   s
  | dd}| dd}t|}|d   }t|d  oP|dkpPtj|k }|r|D ]>}	|	 rZ|	  |krZt	d| d| d|	   qZt
|D ]4\}
}	|	 rt|	s| jd	|	tj| d
||
< q| j|f||}|r| jd	|tj| d
}|S )a  Some PyTorch operators (e.g., Clip/Min/ReLU/Pad) are super set of ONNX in terms of data types.
    This function maximizes the exportability of PyTorch-ONNX by allowing ONNX-unsupported PyTorch
    operator data type. For example, `Cast<int>(Clip<float>(Cast<float>(INPUT)))` can be used to mimic
    `Clip<int>(INPUT)` (opset version < 12).

    Args:
        g (torch._C.Graph): graph to write the ONNX representation into.
        op_name (str): operator name in ONNX.
        *args (tuple): operands to the operator.
        **kwargs (dict): attributes to the operator along with "opset_before" (optional, None by default)
            indicating the smallest opset version to trigger such casting behavior and "target_float_t"
            (optional, "Float" by default) indicating the data type of internal operator.

    Returns:
        Optional[torch._C.Value, Tuple[torch._C.Value, ...]]: output(s) of the operator.
    opset_beforeNtarget_float_trV   r   z
Inputs of z must have same dtype. Got z and rS   rU   )popr   rX   rY   r   rZ   r	   export_onnx_opset_versionZisCompleteTensorrQ   	enumerater   rW   )r   Zop_namerA   r   rB  rC  inputsZdtype_0Zrequire_castr   rr   r#   r   r   r   op_with_optional_float_cast;  s0    rH  c                 C   s   t | d|ddS )NRelu   rB  rH  r   r   r   r   relul  s    rM  c                 C   s   t | d|dd}t| |dS )NrI  rJ  rK     )rH  	clamp_max)r   r   rM  r   r   r   relu6q  s    rP  c                 C   s   |  d|S )NCeilr   r   r   r   r   ceilw  s    rR  c                 C   s   |  d|S )Nr]   r   r   r   r   r   rM   {  s    rM   c                 C   s.   t | || jdtdgd}t| |dgS NrF   r   rG   )r   r   rH   r   r   r-  )r   r#   Zsz_0r   r   r   _len  s    rT  c                 C   s@   t |dkrt ddS t |dkr4t ddS | d|S )Nr   	thresholdznon-zero thresholdznon-zero valuerI  )r   r1   r3   r   )r   r#   rU  rJ   r   r   r   rU    s
    rU  c                 C   s$   t |dd}| jd|t |dS )Nr}   negative_slope	LeakyRelur   )r   r   r   r1   )r   r   rV  Zinplacer   r   r   
leaky_relu  s    rY  c                 C   sP   t ||}|d k	r$|d dks$t| jd||dd\}}| d|| d|S )Nr   r   r  )r   r  r:   r   )r   r  ro   r   )r   r   rv   r<  firstsecondr   r   r   glu  s
    r\  c              
   C   sZ  t |}|d k	r|dk r"|| }||d k}|rptt|}|d ||  ||< |d< | jd||d}|d }| jd||d}|r|  dkrt |d	d
}| jd|t j| d}|r| jd||d}|S | d|| jd||gdd}| d|}	t j	| |	|gd}
| d|	|
}|rV|  dkrVt |d	d
}| jd|t j| d}|S )Nr   r+   r   r   r   ZSoftmaxr/  r
   rr   r_   rS   rU   r7   	ReduceMaxr   Expr:  rR   )
r   r   r   r   r   r   r   r   r   _reducesum_helper)r   r   rv   r_   	input_dimis_transpose_requiredr   softmaxparsed_dtypeexpr   r   r   r   rb    sB    
    rb  c                 C   s@   t |d}|dkr4| d| d| d|||S | d|S )NrD   r+   rR   r@  r:   )r   r   r   )r   r#   r   rU  Z
beta_constr   r   r   softplus  s     re  c                    s   t | }|d k	r$|t d  nd d ksBtdd D rNt tdS fddtdtD   fddtdt D   fddtdtD fddtdtD S )	Nc                 S   s   g | ]}|d kqS r    r   r|   rr   r   r   r   r     s     z)get_pool_ceil_padding.<locals>.<listcomp>input size not accessiblec              	      sB   g | ]:}t t | d |   |  t|  d qS r   r+   )r  mathrR  rm   rf  )rv   kernel_sizepaddingstrider   r   r     s   0r   c                    sD   g | ]<} | d  |  | |  kr8 | d  n | qS r+   r   rf  )ceiled_output_dimrv   rk  rl  r   r   r     s   "c                    sP   g | ]H}| d krdn2| | d|    | d  |  d    qS )r+   r   r   r   rf  )rn  rv   rj  rk  rl  r   r   r     s   
*c                    sd   g | ]\}| d |    | krT|  | d k rDt | q^t  | d n
t | qS rh  r  rf  )rj  rk  padding_ceilr   r   r     s   )r   _get_tensor_sizesr>   anyr3   r   r   )r   rj  rl  rk  sizesr   )rn  rv   rj  rk  rp  rl  r   get_pool_ceil_padding  s"    


rt  c              	      s>   t ddddddt dddddd fdd}|S )NTFrC   r   rr   c                    s2  t |dhkrt dS |s&|}t|}|rbt||||}|tdd t||D  }n|d }|||d}r| jd|fddi|\}	}
| jd|dd	d
 tD dd
 tD d\}}tj| |dd
 tD ddd}t	| |
|}
|	|
fS | jd|fddi|}	|	S d S )Nr+   dilationc                 s   s   | ]\}}|| V  qd S r    r   r|   ar  r   r   r   	<genexpr>  s     z1_max_pool.<locals>.symbolic_fn.<locals>.<genexpr>r   )kernel_shape_ipads_i	strides_iMaxPoolr  c                 S   s   g | ]}d qS rm  r   r|   _r   r   r   r   1  s     z2_max_pool.<locals>.symbolic_fn.<locals>.<listcomp>c                 S   s   g | ]}d qS rm  r   r}  r   r   r   r   2  s     )r  ry  r{  c                 S   s   g | ]}d | qS )r   r   rf  r   r   r   r   8  s     r   r.  )
setr   r3   tuplert  zipr   r   r0  r6   )r   r   rj  rl  rk  ru  	ceil_moderp  r   rr  r~  Zflattened_indicesrL   r   ndimsreturn_indicestuple_fnr   r   symbolic_fn  sB    
z_max_pool.<locals>.symbolic_fn)r   quantized_argsr   )r   r  r  r  r  r   r  r   	_max_pool  s    4r  
max_pool1dr+   )r  
max_pool2dr   
max_pool3d   max_pool1d_with_indicesmax_pool2d_with_indicesmax_pool3d_with_indicesc                    sf   t dt dddddddd
tjttdf ttdf ttttdf f ttd fdd	}|S )NTrC   r   rr   r   .)r   rj  rl  rk  r  count_include_padc                    s   |s|}t |||| }|}|rL| jd|d| d ddd}dt| }|r|t||||}	|tdd	 t|	|D  }n|d }| jd
||||d}
|
S )NPad)r   r   r   constantr   )rz  mode_svalue_fr   c                 s   s   | ]\}}|| V  qd S r    r   rv  r   r   r   rx    s    z1_avg_pool.<locals>.symbolic_fn.<locals>.<genexpr>AveragePool)ry  r{  rz  )r   Z_avgpool_helperr   r>   rt  r  r  )r   r   rj  rl  rk  r  r  Zdivisor_overrideZadjusted_paddingrp  outputr   r  r   r   r  c  sD         

z_avg_pool.<locals>.symbolic_fn)N)r   r  r   r   Valuer   r  r   )r   r  r  r   r  r   	_avg_poolb  s    	 

)r  
avg_pool1d
avg_pool2d
avg_pool3dc                    s"   t dd fdd}|S )NTFc              	      s  zt dW n tk
r.   t d Y S X dgt krVdkrV| d|S t |}z|dd   W n tk
r   d  Y nX  d kstdd  D r҈dgt kr| d	|d fS t d
S  fddt	dt D }|dgt| kr4dgt kr(| d	|d fS t dS  fddt	dt D }dkr| |||dt  dt  dS | j|||d}|S )Nr   z4adaptive pooling, since output_size is not constant.r+   r  ZGlobalAveragePoolr   c                 S   s   g | ]}|d kqS r    r   rf  r   r   r   r     s     z7_adaptive_pool.<locals>.symbolic_fn.<locals>.<listcomp>ZGlobalMaxPoolrg  c                    s   g | ]} | |  qS r   r   rf  rv   output_sizer   r   r     s     r   z-output size that are not factor of input sizec                    s    g | ]}t  | |  qS r   ro  rf  r  r   r   r     s     r|  r  rm  F)ry  r{  )
r   
_parse_arg	Exceptionr   r>   r   rq  rr  r3   r   )r   r   r  rs  rf   kr  r   r   r  rX   r  r   r    s<    


 
$z#_adaptive_pool.<locals>.symbolic_fn)r   r  )r   rX   r  r   r  r   r  r   _adaptive_pool  s    
*r  adaptive_avg_pool1dr  adaptive_avg_pool2dadaptive_avg_pool3dadaptive_max_pool1dr|  adaptive_max_pool2dadaptive_max_pool3dc                 C   sT   t | tstt|d d  dg| d t|   }|dd d |dd d  }|S )Nr   r   r   )r   r  ro   r   r>   )rv   padpaddingsr   r   r   _prepare_onnx_paddings  s    &r  c              	   C   sf   t | d} t | rbt | rbt | }zdd |D } W n$ tk
r`   t dddd Y S X | S )Nr   c                 S   s   g | ]}t |d dqS )rr   rk  )r   r   )r|   rC   r   r   r   r     s    z)_convert_padding_node.<locals>.<listcomp>r  r)   r*   z)The sizes of the padding must be constant)r   r   r.   r   rt   r  r0   )rk  
input_listr   r   r   _convert_padding_node  s    

   
r  c              	   C   sj   d}zt |dd}W n$ tk
r:   t dddd Y S X t|}tt ||}t| d||||ddS )	Nr  rD   rJ   r  r)   r*   z*The value for the padding must be constant)rz  r  r  rB  )r   r   r  r0   r  r  r   rH  )r   r   rk  rJ   r  r  r   r   r   constant_pad_nd  s*       
      r  c              
   C   sB  t |}t|d dkstt|d }|}t|D ]}|d| d   }|d| d   }g }	|dkrtj| |d| g|d  gdgd}
|	|
 |dk s|dk rtj| |d| gtd| gdtd|   gd}|	| n
|	| |dkr$tj| |d| gdg|gd}|	| | jd|	dd| i}q4|S )	Nr   r   r+   r   r.  r   r   )r   )	r  r>   ro   r   r   r0  r   maxr   )r   r   r  rk  ndimcuridxZpad_lZpad_rrw   leftZmiddlerightr   r   r   _pad_circular  sL       
 


    
r  c                 C   s0   d}t |}tt||}t| d|||ddS )Nreflectr  r*   rz  r  rB  r  r  r   r   rH  r   r   rk  r  r  r   r   r   reflection_pad;  s         r  c                 C   s0   d}t |}tt||}t| d|||ddS )NZedger  r*   r  r  r  r   r   r   replication_padD  s         r  c                 C   sp   t |d}|dkr t| ||S |dkr4t| ||S |dkrJt| |||S |dkr^t| ||S td| d S )NrL   Z	replicater  r  ZcircularzUnrecognized padding mode )r   r  r  r  r  r  rQ   )r   r   r  r  rJ   r   r   r   r  U  s    r  c                    s    fdd}|S )Nc                    s`   t | |\}}t  t |}|r6t dS |d krNt | || }| jd||dS )Nzalign_corners == TrueUpsampler  )r   Z_get_interpolate_attributesZ_interpolate_warningr2   r3   Z_interpolate_size_to_scalesr   )r   r   r  rA   scalesalign_cornersrv   interpolate_moder   r   r   r  d  s"      

   z!_interpolate.<locals>.symbolic_fnr   )r   rv   r  r  r   r  r   _interpolatec  s    r  upsample_nearest1dZnearestupsample_nearest2dr  upsample_nearest3d   upsample_linear1dlinearupsample_bilinear2dupsample_trilinear3dc           	      C   s*   t | |||||\}}| jd|||dS )Nr  r  )r   Z _interpolate_get_scales_and_moder   )	r   r   r   Zscale_factorr  r  Zrecompute_scale_factorZ	antialiasr  r   r   r   __interpolate}  s         r  c                 C   s$   |   dkrtd| d|S NBoolzOONNX export does NOT support exporting bitwise Not for non-boolean input valuesrb   rX   rY   r   r   )r   inpr   r   r   bitwise_not  s
    r  c                    s    fdd}|S )Nc                    s    fdd}|S )Nc                    s   | j d | ||tj dS NrS   rU   )r   r   rW   r   r   r&   r   to_typer   r   wrap_with_cast  s
    
zGwrap_logical_op_with_cast_to.<locals>.decorator.<locals>.wrap_with_castr   r   r  r  r   r   	decorator  s    z/wrap_logical_op_with_cast_to.<locals>.decoratorr   r  r  r   r  r   wrap_logical_op_with_cast_to  s    
r  c                    s    fdd}|S )Nc                    s    fdd}|S )Nc                    s@   t  d }t|   }|| || |d|| |dS )N_cast_{}F)globalsr   r  rX   rY   )r   r   r&   Zto_cast_funcZfrom_cast_funcr  r   r   r    s     
 
zPwrap_logical_op_with_cast_to_and_from.<locals>.decorator.<locals>.wrap_with_castr   r  r  r   r   r    s    z8wrap_logical_op_with_cast_to_and_from.<locals>.decoratorr   r  r   r  r   %wrap_logical_op_with_cast_to_and_from  s    
r  c                    s    fdd}|S )Nc                    s   |  d | ||S )Nrb   r   r  funcr   r   wrap_with_not  s    z4wrap_logical_op_with_negation.<locals>.wrap_with_notr   )r  r  r   r  r   wrap_logical_op_with_negation  s    r  c                 C   s$   |   dkrtd| d|S r  r  rp   r   r   r   __not_  s
    r  c                 C   sH   t | tjr:t | tjr:| jdtjdtjddS | d||S )NrF   Tr^   rG   rc   )r   rX   r   DeviceObjTyper   rH   rI   boolr;   r   r   r   eq  s     r  c                 C   s   t | ||S r    )r  r;   r   r   r   ne  s    r  c                 C   s   t | ||S r    gt_implr  r   r   r   gt  s    r  c                 C   sz   |   d k	rl|   dkrl|   d k	rl|   dkrl| jd|tjd d}| jd|tjd d}| d||S )Nr  rS   IntrU   ZGreaterrX   rY   r   r   rW   r  r   r   r   r    s    r  c                 C   s   t | ||S r    lt_implr  r   r   r   lt  s    r  c                 C   sz   |   d k	rl|   dkrl|   d k	rl|   dkrl| jd|tjd d}| jd|tjd d}| d||S )Nr  rS   r  rU   Lessr  r  r   r   r   r    s    r  c                 C   s   t | ||S r    r  r  r   r   r   ge  s    r  c                 C   s   t | ||S r    r  r  r   r   r   le  s    r  c                 C   s:   |   dkr.|   dkr.| d||S tdd S )Nr  ra   zOONNX export does NOT support exporting bitwise AND for non-boolean input valuesrX   rY   r   r   r  r   r   r   __and_  s
     r  c                 C   s:   |   dkr.|   dkr.| d||S tdd S )Nr  OrzNONNX export does NOT support exporting bitwise OR for non-boolean input valuesr  r  r   r   r   __or_  s
     r  c                 C   s:   |   dkr.|   dkr.| d||S tdd S )Nr  r`   zOONNX export does NOT support exporting bitwise XOR for non-boolean input valuesr  r  r   r   r   __xor_  s
     r  r  c                 C   s   |  d||S )Nra   r   r  r   r   r   logical_and  s    r  c                 C   s   |  d||S )Nr  r   r  r   r   r   
logical_or  s    r  c                 C   s   |  d||S )Nr`   r   r  r   r   r   logical_xor  s    r  c                 C   s   |   |   kr6| jd|tj|    d}| jdtjdtjdd}t|sp| jd|tjd d}| d||}| jd|tj|    d}| d	||}|S )
NrS   rU   rF   r   r^   rG   rV   PowrR   	rX   rY   r   r   rW   rH   rI   Zfloat32rZ   )r   r#   r&   twotwo_powrshiftr   r   r   	__rshift_!  s"    
r  c                 C   s   |   |   kr6| jd|tj|    d}| jdtjdtjdd}t|sp| jd|tjd d}| d||}| jd|tj|    d}| d	||}|S )
NrS   rU   rF   r   r^   rG   rV   r  r:   r   )r   r#   r&   r  r  lshiftr   r   r   	__lshift_9  s"    
r  c              	   C   sh   |   dkr&| jd|tjd d}|d krXt| |}t| || jdtdd|S | d|||S )Nr  rS   rU   rF   r+   rG   Where)	rX   rY   r   r   rW   nonzeroZ_unbind_helperrH   rI   )r   	conditionr#   r&   r!  r   r   r   r   Q  s      
   r   c           	      C   s   t |}|d krt ddS |dk r.|| }||d k}|r|tt|}|d ||  ||< |d< | jd||d}|d }| jd||d	}|r|  d
krt |dd}| jd|t j	| d}|r| jd||d}|S )Nrv   fONNX and PyTorch use different strategies to split the input. Input rank must be known at export time.r   r+   r   r   r   Z
LogSoftmaxr/  r
   rr   r_   rS   rU   )
r   r   r3   r   r   r   r   r   r   r   )	r   r   rv   r_   r`  ra  r   Z	return_oprc  r   r   r   log_softmax`  s2    
  r  c                 C   s  t |}z|dd  }W n tk
r2   d }Y nX |d ksNtdd |D rVtd||g}t |st |dkr|| |dd  ||| ||	d}tdd |D r|stt	|t	|kst||d	< | j
|rd
ndf||}t |st |dkr| 
d||S |S d S )Nr   c                 S   s   g | ]}|d kqS r    r   rf  r   r   r   r     s     z _convolution.<locals>.<listcomp>zDUnsupported: ONNX export of convolution for kernel of unknown shape.r+   )ry  r{  rz  Zdilations_iZgroup_ic                 s   s   | ]}|d kV  qdS r   Nr   )r|   or   r   r   rx    s     z_convolution.<locals>.<genexpr>Zoutput_padding_iZConvTransposeZConvr(   )r   rq  r  rr  rQ   r   r   r   ro   r>   r   )r   r   r  biasrl  rk  ru  Z
transposedoutput_paddinggroupsZ	benchmarkZdeterministiccudnn_enabledZ
allow_tf32Zweight_sizeZkernel_shaperA   r   r   r   r   r   _convolution  s@    




r  c                 C   s"   t | ||||||dd|d d d d S NFr   r  r   r   r  r  rl  rk  ru  r  r   r   r   conv1d  s     r  c                 C   s"   t | ||||||dd|d d d d S r  r  r  r   r   r   conv2d  s     r  c                 C   s"   t | ||||||dd|d d d d S r  r  r  r   r   r   conv3d  s     r  c	           	      C   s"   t | ||||||d||d d d d S NTr  	r   r   r  r  rl  rk  r  r  ru  r   r   r   conv_transpose1d  s     r  c	           	      C   s"   t | ||||||d||d d d d S r  r  r  r   r   r   conv_transpose2d  s     r  c	           	      C   s"   t | ||||||d||d d d d S r  r  r  r   r   r   conv_transpose3d+  s     r  c
                 C   s   t |d t rBt |||||gsBtjdk rBt ddddS t | |||||\}}}}| j	d||||||d| |s~dndd	}
|s|
S |
\}}}}}|
|  |
|  |d	|   |d	|   |S d S )
N
batch_norm   ZBatchNormalizationr)   zaAll input tensors must have the same `dtype`. Turn off Autocast or export using opset version 15.r+   r  )	epsilon_fZ
momentum_fr  zbatch_norm_dead_output-)r   check_training_moderH   Zis_autocast_enabledZargs_have_same_dtyper	   rE  r0   Z_batchnorm_helperr   r   rX   ZsetDebugNameZ	debugName)r   r   r  r  running_meanrunning_vartrainingmomentumepsr  r[   resZnew_running_meanZnew_running_varZ
saved_meanZ	saved_varr   r   r   r  A  sT         
r  c              	   C   s   t  r | jd||||||dS dd tt|ddD }t | d}t | |}	| jd||d	}
t| ||
}| jdt| |||d	}t	| t
| ||	}| d
||}|d kst |st| ||}|d kst |st
| ||}|S )N
layer_norm)Znormalized_shape_ieps_fZcudnn_enable_ic                 S   s   g | ]
}| qS r   r   rf  r   r   r   r     s     zlayer_norm.<locals>.<listcomp>r   r          @r   r:  rR   )r   r   r   r   r>   Z_generate_wrapped_numberr   r6   powr   r,   r   r<   )r   r   Znormalized_shaper  r  r&  Zcudnn_enabler   Ztwo_cstZeps_cstr   	numeratorZvariancedenominatorr(  r   r   r   r(  x  s.    
r(  c
                 C   s.  t |d t |d}
|d ks*t |rn|
d kr:tdtdg|
 d|   d }| j	d|d}|d kst |r|
d krtdtd	g|
 d|   d }| j	d|d}|d kst |s|d kst | r| j	d
||||dS t 
|}| }|d }|d kr*td|d }d|d< || |d< t| || j	dtj|gtjdd}t| || j	dtj|gtjdd}t| || j	dtj|gtjdd}t| || j	dtj|gtjdd}| 	d|| j	dt|d}t| |||||||||	
}t| || j	dt|dS d S )Ninstance_normr+   zCUnsupported: ONNX export of instance_norm for unknown channel size.rE   torch.r   rF   rG   r   InstanceNormalizationr   r   zJUnsupported: ONNX export of instance_norm training for unknown batch size.r^   ZReshape)r   r!  r  r   rQ   rH   rI   rX   rY   r   rq  copyrepeatrd   r   r  r  )r   r   r  r  r"  r#  Zuse_input_statsr%  r&  r  channel_sizeweight_value
bias_value
input_sizeZinput_size_reshaper   cweight_bias_Zrunning_mean_Zrunning_var_input_reshapedr[   r   r   r   r.    s    

    r.  c                    s   t  rjd ||dS t }z|  }W n tk
rJ   d }Y nX |d k	rtd||}t||d |} fddt||D }	t|}
ttd|
	
   fdd|	D }jd|d	 iS t d
dS d S )Nunfold)Zdimension_iZsize_iZstep_ir   r+   c              	      s*   g | ]"\}}t j g|g|gd qS )r.  r   r0  )r|   lowhi)	dimensionr   r   r   r   r     s       zunfold.<locals>.<listcomp>c              
      s(   g | ] }t jd |d gqS )r   r   )r   rz   r   r{   )r@  r   permr   r   r   
  s     r   r   ZUnfoldrg  )r   )r   r   r   rq  r  r   r  r>   r   r   rD  r   r3   )r   r   r@  r   steprs  ZsizedimZlow_indicesZ
hi_indicesr   r  r   r   )r@  r   r   rA  r   r<    s*    

r<  c                 C   sF   |r|dkrt ddS |r0|dkr0t ddS | jd|t |dS )NrE   r   zdoes not support scale in Eluinput_scalez#does not support input_scale in EluElurX  )r   r3   r   r1   )r   r   r5   r   rC  r   r   r   elu  s     rE  c                 C   s   |  d|S )NZSelur   r   r   r   r   selu!  s    rF  c                 C   s   t | |||S r    )r   _select_helper)r   r#   rv   r   r   r   r   index_select%  s    rH  c                 C   s   t |rt |}n|g}t  rD|g| ||g }| jd| S t |d}t|dkrr|rlt| ||S |S nt ddd d S )N	index_putr  r   r)   r*   )rI  )	r   r   rt   r   r   r  r>   r,   r   )r   r#   Zindices_list_valuevalues
accumulateZindices_listrA   r   r   r   rI  -  s    
rI  c           	      C   st   t |d}t  r*| jd|||d|dS t | |||\}}t |}t | ||}t| ||d }t| ||||S )Nrr   
index_fillZ
int_Scalar)r  r   )	r   r  r   r   _index_fill_reshape_helperr2   r   r   scatter)	r   r#   rv   r   rJ   	dim_valueexpanded_index_shapeexpanded_indexZexpanded_valuer   r   r   rL  A  s(    	   
rL  c                 C   sL   t |d}t  r(| jd||||dS t | |||\}}t| ||||S )Nrr   
index_copyr   )r   r  r   r   rM  rN  )r   r#   rv   r   sourcerO  rP  rQ  r   r   r   rR  W  s       rR  c                 C   s   t jj}|rt jj}| jd| d|| d|dd}ttdt|d }t	| t
| |||d }|rxt| ||}	nt| ||}	| jd|	|d}
tj| |
dgddS )	Nr   r   r   r/  r+   rS   rU   r   )_C_onnxZTensorProtoDataTyper   ZINT32r   r   r   r   r   r   rz   r  r  r_  )r   r#   Z
boundariesZ	out_int32r  Zout_type	new_shapeZunsqueeze_axesZexpanded_boundariescondZcond_outr   r   r   	bucketizea  s     "
rW  c                 C   sh   t |}t |}||kr(|d k	r(|S |d k	rF| jd|t j| dS t  r\| d||S tdd S )NrS   rU   type_aszUnsupported: ONNX export of type_as for tensor of unknown dtype. Please check if the dtype of the parameter passed to the type_as function is correct.)r   r   r   rW   r   r   rQ   )r   r#   r&   r   Zother_dtyper   r   r   rX    s    

  rX  c           	      C   s   t  r| jd||||dS t j| t| |||gdd}t j| t| |||gdd}t j| t| |||gdd}t| t| t| ||| jdt	|gd}t
| ||S )Ncosine_similarity)r   r)  r   r   rF   rG   )r   r   r   r_  r<   r  r   r   rH   rI   rB   )	r   x1x2rv   r&  crossZx1_l2Zx2_l2Zdiv_tensr   r   r   rY    s4     
   
   
    rY  c                 C   s   t |s | jdt|gd}t| | jdtjdgtjddt| ||}t j| t	| t
| |||dgt |dd}t	| ||S )NrF   rG   r+   r^   r   rr   r   )r   r.   r   rH   rI   rB   rm   r,   r_  r+  r6   r  )r   Zinput1Zinput2pr&  r   Zinv_pZ	summationr   r   r   pairwise_distance  s    


r^  c                 C   s   |S r    r   )r   r   Zunused_memory_formatr   r   r   clone  s    r_  c                 C   s   |  d|S )NAbsr   rp   r   r   r   abs  s    ra  c                 C   s   |  d|S )NLogr   rp   r   r   r   log  s    rc  c              
   C   s"   t | t| t| td||S )Nr+   )rc  r,   r   r   rH   r   rp   r   r   r   log1p  s     rd  c              	   C   s*   d}|  dt| || j dt|gdS )NgUk@rR   rF   rG   r   rc  rH   rI   )r   r#   Z_ln10r   r   r   log10  s    rf  c                 C   sf   |    }}t|s4d}| jd|tj| d}t|sT| jd|tj| d}| d||}|S )NrV   rS   rU   r  )rX   rY   r   rZ   r   rW   )r   r#   exponentZf_dtyper   r+  r   r   r   r+    s    

  r+  c              	   C   s|   t |rt| ||S t |r,t| ||S t |rdt |rdt| d|t |dt |dddS t| t| |||S d S )NCliprD      min_fmax_frB  )r   r   rO  	clamp_min_is_constantrH  r  )r   r#   minr  r   r   r   clamp  s    



	rp  c                 C   s\   t |r$t| d|t |dddS |  }| jd|t j| d}t| d||ddS d S )	Nrh  rD   ri  )rk  rB  rS   rU   MaxrK  r   rn  rH  r  rX   rY   r   rW   )r   r#   ro  r_   r   r   r   rm    s    
   
 rm  c                 C   s\   t |r$t| d|t |dddS |  }| jd|t j| d}t| d||ddS d S )	Nrh  rD   ri  )rl  rB  rS   rU   MinrK  rr  )r   r#   r  r_   r   r   r   rO   	  s    
   
 rO  c                 C   s   |d kr |d kr | j d|ddS |d kr:t| d||ddS t|dd}t|dd	}| j d||g|d
}| j d|||d}||fS d S )Nr]  r   r   rq  ri  rK  rr   rv   r   r   ArgMaxr   r   r   rH  r   r   )r   r#   dim_or_yr   rv   r  r  r   r   r   r  	  s    r  c                 C   s   t | ||dS N)rx  )r  r  r   r   r   maximum	  s    rz  c                 C   s   |d kr |d kr | j d|ddS |d kr:t| d||ddS t|dd}t|dd	}| j d||g|d
}| j d|||d}||fS d S )N	ReduceMinr   rt  rs  ri  rK  rr   rv   r   r   ArgMinrv  rw  )r   r#   rx  r   rv   ro  r  r   r   r   ro  "	  s    ro  c                 C   s   t | ||dS ry  )ro  r  r   r   r   minimum2	  s    r}  c                 C   s   | j d|||dS )Nr]  r   r   r   r#   rv   r   r   r   r   amax6	  s    r  c                 C   s   | j d|||dS )Nr{  r   r   r~  r   r   r   amin;	  s    r  c                 C   sJ   d|i}t |s*t |dd}|g|d< | jd|f|| jd|f|fS )Nr   rr   rv   r   r{  r]  )r   r   r   r   )r   r#   rv   r   Zreduce_kwargsr   r   r   aminmax@	  s    

 r  c                 C   s   |  d|S )Nr^  r   rp   r   r   r   rd  L	  s    rd  c                 C   s8   t |d |s|S td | jd||dd\}}|S )NdropoutzDropout is a training op and should not be exported in inference mode. For inference, make sure to call eval() on the model and to export it with param training=False.ZDropoutr   )Zratio_fr  )r   r!  r  r  r   )r   r   r]  trainr  r~  r   r   r   r  P	  s    r  c                    s   t ddd fdd}|S )NrC   rD   rr   c                    s   |rt  dS |S )Nztraining mode)r   r3   )r   r   r]  r  r   r   r   feature_dropout_	  s    z-_unsupported_dropout.<locals>.feature_dropoutr   )r   r  r   r  r   _unsupported_dropout^	  s    r  r  alpha_dropoutfeature_alpha_dropoutc                 C   s<   |dkrt d}n|dkr$t d}ntd|| |||dS )Nr+   ZReduceL1r   ZReduceL2z)ONNX export only p-norms with p of 1 or 2)rv   r   )r   rQ   )r   r#   r]  rv   r   rD   r   r   r   normt	  s    

r  c              	   C   s~   t  r| jd||||dS | jd|dddgd}| jd|dddgd}t| |||dg|gdgd}| jd|dddgdS d S )Nconv_tbc)Zpad_ir   r+   r   r   r   )r   r   r   r   r  )r   r   r  r  r  convr   r   r   r  	  s    r  c                 C   s*   t  r| jd|||ddS t dS d S )N_uniquer   )sorted_ireturn_inverse_ir  )r   r   r   r   )r   r   sortedreturn_inverser   r   r   r  	  s    r  c                 C   s0   t  r| jd||||ddS t ddd d S )N_unique2r  )r  r  Zreturn_counts_ir  r)   r*   )r   r   r   r   )r   r   r  r  Zreturn_countsr   r   r   r  	  s    	r  r  c                 C   s   t | |||||S r    )zeros)r   rs  r_   layoutdevice
pin_memorymemory_formatr   r   r   empty	  s    r  c                 C   s   t | |||||S r    )
zeros_like)r   r   r_   r  r  r  r  r   r   r   
empty_like	  s    r  c                 C   sB   t |}|d kr0|d k	r0|}t jt j| }t| |||||S r    )r   r   r   r   rW   r  r   r#   rs  r_   r  r  r  r   r   r   r   	new_empty	  s    
r  c                 G   s8   t |dd}|d krt jj}| jd|t j| d}|S )Nrr   r_   rS   rU   r   r   r   FLOATr   r   )r   Zscalarr_   optionsr   r   r   scalar_tensor	  s
    r  c                 C   s(  t |dd}t |r|d krHt |d   }t jt j| }t	 }t |D ]H}| j
dtdgd}t | ||}| j
d|t j| d}|| qX| j
d|d
diS |d kr|  }t jt j| }t |rt |s t |r| j
d|ddd}| j
d|t j| dS )Nrr   r_   r   rF   r+   rG   rS   rU   r   r   ZConcatFromSequence)r   Z
new_axis_i)r   )r   r   r   rt   rX   rY   r   r   rW   r   r   rH   r   r!   r   Z_is_listr/   Z_is_scalar_list)r   datar_   r  requires_gradr  r}   Zshape_referencer   r   r   rI   	  s4    
rI   c                 C   s   t | |||S r    )rI   )r   r  r_   r  r   r   r   	as_tensor	  s    r  c                 C   sp   |d krt jj}t |d}t|trNt|dkrN| jdt	g 
tjd}| jd|tj	dgt j| ddS )Nr   r   rF   rG   ConstantOfShaper^   r   r   r  r   r   r   r>   r   rH   rI   r   rd   r   r   rs  r_   r  r  r  sizes_r   r   r   r  	  s     r  c                 C   s>   |  d|}|d krtjj}| j d|tjdgtj| ddS )Nr   r  r   r^   rG   r   r   r   r  rH   rI   r   r   r   r_   r  r  r  r  r   r   r   r   r  
  s     r  c                 C   sB   t |}|d kr0|d k	r0|}t jt j| }t| |||||S r    )r   r   r   r   rW   r  r  r   r   r   	new_zeros
  s    
r  c                 C   sp   |d krt jj}t |d}t|trNt|dkrN| jdt	g 
tjd}| jd|tj	dgt j| ddS )Nr   r   rF   rG   r  r+   r^   r  r  r   r   r   r    
  s     r   c                 C   s>   |  d|}|d krtjj}| j d|tjdgtj| ddS )Nr   r  r+   r^   rG   r  r  r   r   r   r   0
  s     r   c                 C   sB   t |}|d kr0|d k	r0|}t jt j| }t| |||||S r    )r   r   r   r   rW   r   r  r   r   r   new_ones@
  s    
r  c           
   	   C   s   t |d}t |rX|d kr&t jjn|}t| ||||}t| ||| jdt	ddS t 
|dd}|d krvt jjn|}t |d}	t|	trt|	dkr| jdt	g tjd}| jd	||dt j| dS d S )
Nr}   rF   r+   rG   rr   r_   r   r   r  )r   r   r.   r   r  r  r,   r   rH   rI   r   r   r   r>   r   rd   r  r   )
r   rs  rJ   r_   r  r  r  const_valuetmpr  r   r   r   fullJ
  s"    

r  c           
   	   C   s   t |d}t |dd}|d kr*t jjn|}t |r|t| ||||}| jd|t j| d}t	| ||| jdt
ddS | d	|}	| jd
|	t
|gt j| dS d S )NrD   rr   r_   rS   rU   rF   r+   rG   r   r  )r   r   r   r   r  r.   r  r   r   r,   rH   rI   r   r   )
r   r   
fill_valuer_   r  r  r  r  r  r   r   r   r   	full_like_
  s&    

  r  c           	      C   sD   t |}|d kr0|d k	r0|}t jt j| }t| ||||||S r    )r   r   r   r   rW   r  )	r   r#   r   r  r_   r  r  r  r   r   r   r   new_full}
  s    
r  c                 G   s   t |dkrX|\}}}}}t| |dg}| jd||dd}t| ||||}	| d|	S t |dkr|\}}
}}}}| jdt| |dgt| |
dgdd}t| ||||}	| d|	S tdd S )Nr  r   r   r/  ZEyeLikerN  zUnknown aten::eye signature)r>   r   rz   r   r  r   )r   rA   r   r_   r  r  r  r<  r   rI   mr   r   r   eye
  s"    r  c                 G   s  t |dkrh|\}}}}t|d}|dkr6td|  dkoT|  dk}|  dkot|  dk}|  dk}	|  dk}
|s|	r|s|
r|  dkrtjt	j
jkrtdnBt| |d	g}t| |d	g}t| |d	g}| d
||||S nR|rd	n
t|d}|r4dn
t|d}t|d}tj| ||g|g|gdS nt |dkr|\}}}d	}|  dko|  dk}|  dko|  dk}|rd	n
t|d}|rdn
t|d}tj| ||g|g|gdS tdd S )Nr  rr   r+   z"step!=1 is currently not supportedr
   ZNoneTyper   zUnsupported: ONNX export of Slice with dynamic inputs. DynamicSlice is a deprecated experimental op. Please use statically allocated variables or export to a higher opset version.r   ZDynamicSlicer   r.  r  zUnknown aten::slice signature)r>   r   r  rQ   r   r   rX   r	   operator_export_typerT  ZOperatorExportTypesZONNXrz   r   r0  r   )r   r#   rA   rv   startendrB  Zis_start_noneZis_end_noneZis_start_onnx_constZis_end_onnx_constZstart_unsqueezedZend_unsqueezedZdim_unsqueezedr   r   r   slice
  s    
    
     r  c                 C   s   t | d|||ddS )Nrh  ri  rj  rL  )r   r#   Zmin_valZmax_valr   r   r   hardtanh
  s         r  c                 C   s   t | |}| d||S r9   )hardsigmoidr   )r   r#   Zhsr   r   r   	hardswish
  s    
r  c                 C   s   | j d|ddS )NHardSigmoidgUUUUUU?rX  r   rp   r   r   r   r  
  s    r  c                 C   s   |  d|t| |S )Nr7   )r   r   rp   r   r   r   
tanhshrink
  s    r  c              
   C   sZ   | j dt|gd}t| t| ||t| |t| |}|  d||| j dtdgdS NrF   rG   r  r   )r   rH   FloatTensorr  r  r  r   )r   r#   lambdlambd_oprV  r   r   r   
hardshrink   s    "r  c              
   C   s   | j dt|gd}t| ||}|  d|t| ||| j dtdgd}t| |t| |}|  d|t| ||| j dtdgd}t| ||S r  )r   rH   r  r  r6   r  r   r,   )r   r#   r  r  Zgt_condZgt_outZlt_condZlt_outr   r   r   
softshrink  s     

r  c                 C   s   |S r    r   rp   r   r   r   alias  s    r  c                 C   s|   |dk rjt |}|d k	r^tdt| d d d t|| d  d d  || d }nt d	d
S t j| ||gdS )Nr   z)ONNX export unsqueeze with negative axis r3  r4  r5  r+   r6  r7  r   r9  r:  )r   r   r  r  r;  r3   rz   r  r   r   r   r     s4    

 r   c                 C   sl   |d k	rt dd t |}z|| }W n tk
rB   d }Y nX |d krXt ddS | jd|||ddS )NZSortz'Out parameter is not supported for sortrg  TopKr   Zk_ir   r  )r   r3   rq  r  r   )r   r#   rv   Z	decendingr[   Z
self_sizesr<  r   r   r   sort8  s     

r  c                 C   s   |  d|}| j d|ddS )Nr   r   r   rt  r   r"   r   r   r   numelJ  s    r  c                 C   s8   |d k	rt dd |s$t dd | jd|||ddS )Nr  z'Out parameter is not supported for topkzAscending TopK is not supportedr   r  )r   r3   r   )r   r#   r  rv   largestr  r[   r   r   r   topkO  s     r  c                 G   s  dd }||r|S t |dkr|d }t|d r|d   dkr|d  d }t|tjrt |jdkr|	 }t
|}n|}t|st|tjr|d   }| jd|tj| dS | jd|tj| dS nt |d	krt|d
 dd}| jd|tj| dS t |dkrJt|d dd}| jd|tj| dS t |dkrt|d dd}| jd|tj| dS tdS d S )Nc                 S   s   t | dkrL| d   dkpJ| d  tj pJt| d  tj	S t | dkrrt
| d dd}|d kS t | dkrt
| d dd}|d kS d	S )
Nr  r   prim::devicer  r+   rr   r_   )rN     F)r>   r   r   rX   isSubtypeOfr   ListTypeofIntsr   r  r   r   )rA   r_   r   r   r   is_aten_to_device_only\  s    z"to.<locals>.is_aten_to_device_onlyr  r   r   rJ   rS   rU   r  r+   rr   r_   rN  r  zUnknown aten::to signature)r>   r   r.   r   r   r   rH   r   r   itemr  rX   rY   r   rW   r   r   r   )r   r#   rA   r  r_   Ztvalr   r   r   r   [  s:    
r   c                 C   s0   t jj}t| ||}| d||}| d||S )Nr   ZTile)r   r   r   r   r   )r   r#   repeatsr_   Zshape_r   r   r   r3    s    r3  c              
   C   s  |}t |r4t | || jdtdgd}d}n
t |}t |}t |}t |}|d krlt	d|d kr|t	d|d krt	d|
 }	t|D ] \}
}|d krd\||
< |	|
< q|dks|d	krD|d d	krDt |s| jdt|d}|| dkrt d
dddS || }t| || jdt|gdd }nr|d	kr|| dkrlt d
dddS |d d krt d
dddS |d || kstd|d }nt	dt }t | ||d}t | |||}d\||< |	|< t|D ]\}
}t| ||
 |d	 }| jdt|	d |d	  d|| jdt|	|d	 d  dg}| jd|ddi}t| ||d }t j| || jdt|ddd}|| q| jd|d|iS )NrF   r   rG   r   zGUnsupported: ONNX export of repeat_interleave for unknown repeats rank.zGUnsupported: ONNX export of repeat_interleave for unknown repeats size.zEUnsupported: ONNX export of repeat_interleave for unknown input size.)r   r   r+   repeat_interleaver)      z3Unsupported along dimension with unknown input sizez*Unsupported for cases with dynamic repeatsz2repeats must have the same size as input along dimz%repeats must be 0-dim or 1-dim tensor)r   r+   r   r   Z	allowzero)r   )r   )r   r   r!   r   rH   rI   r2   r   rq  rQ   r2  rF  
_is_tensorr   r0   r   ro   r   Z_repeat_interleave_split_helperr   r   )r   r#   r  rv   r  r   Zrepeats_dimZrepeats_sizesZinput_sizesZinput_sizes_tempr  r7  ZrepsZfinal_splitsZr_splitsZi_splitsZr_splitZi_splitZr_concatr   r   r   r    s    
  



 
   
   
r  c           	      C   s  t |}t|dkr"t ddS tdd |dd  D rt j| t | |ddg| jd	t	d
d||d
d
gdd
d}| jd|d
dddddgd}t j| || jd	t	d
d
ddd
d
gdd
d}t j| || jd	t	d
d
d
d
ddgdd
d}t 
| |ddgS |d | | }t j| || jd	t	d||||d |d gdd
d}| jd|d
dddddgd}t j| || jd	t	d||d | |d | gdd
dS d S )Nr  pixel_shuffleonly support 4d inputc                 s   s   | ]}|d kV  qd S r    r   rf  r   r   r   rx  
  s     z pixel_shuffle.<locals>.<genexpr>r+   r   r  rF   r   r   rG   r  r   r  r   r   rq  r>   r3   rr  r!   rz   r   rH   rI   r-  )	r   r#   Zupscale_factorr  
after_viewafter_transpose	reshape_h	reshape_woutput_channelr   r   r   r    sz    
	

r  c           
      C   s  t |}t|dkr"t ddS tdd |dd  D r t j| t | |dg| jdt	d	d	d
|d	gdd	d}t j| || jdt	d	d	d	d	d
|gdd	d}| jd|d	dddddgd}t j| || jdt	d	d
ddd	d	gdd	d}t 
| |ddgS |d | | }t j| || jdt	d
|d |d | ||d | |gdd	d}	| jd|	d	dddddgd}t j| || jdt	d
||d | |d | gdd	dS d S )Nr  r  r  c                 s   s   | ]}|d kV  qd S r    r   rf  r   r   r   rx  O  s     z"pixel_unshuffle.<locals>.<genexpr>r+   r  rF   r   r   rG   r  r   r  r   r   r  )
r   r#   Zdownscale_factorr  r  r  r  Zfinal_reshaper  r  r   r   r   pixel_unshuffleJ  st    




r  c           *         s  t d d d d d  dddd	d
ddddddg}ttdd |D |}|rXdnddkrt  d|	  krtddS t  d|	  kst fddtdt D |
rވj	d|dddgd}|r|rtddS 
dr|dd    }d d }t|dd kr@tddS |	 }|}g }dksbd krh|}ndkr~|\}}g }|d krtn|}d krd!d"d#gndkrd"d$d%gd&d' fd(d)}fd*d+}fd,d-}tD ]R}|rPdkr.||\}}}n||\}}t}||d f}ndkr|d| \}} }!|d| d \}"}#}$j	d.|!|$dd/}n,|d| \}} |d| d \}"}#t}j	d.||"dd/}j	d.| |#dd/}d| d| d f}|||||g}%|%||f|  dkr:|%||f|  |rDi nd0d1i}&dkr|	rf||g}'n|g}'j	d;|%d|'d2|&\}}(nVd krj	d<|%ddd3|&\}}(n*dkrj	d=|%d4d5|&\}}(})|	r,j	d|dddd4gd}tj|j	d6tddd7gd8dd9}nt|dg}||( dkr||) q|
rzj	d|dddgd}dkr|(nj	d>|d:di}dksd kr||fS dkrdkr|)nj	d?|d:di}|||fS d S )@NzVExporting a model to ONNX with a batch_size other than 1, with a variable length with z can cause an error z9when running the ONNX model with a different batch size. z4Make sure to save the model with a batch size of 1, z=or define the initial states (h0/c0) as inputs of the model. rI  r   r   ZAffinerW  ZThresholdedReluZ
ScaledTanhr  rD  ZSoftsignr@  c                 S   s   g | ]}|  qS r   )lower)r|   Zact_funr   r   r   r     s     z _generic_rnn.<locals>.<listcomp>r  r   LSTMr+   zLSTMs with projectionsc                    s   g | ]} ||  qS r   r   rf  )all_weightsweights_per_layerr   r   r     s   r   r   r   zRNN/GRU/LSTMzdropout in training modeRNNzunknown hidden sizeGRU)r+   r   )r   r+   )r   r  )r  r  )r+   r  c                    s*    fdd|D } j d|ddiS )Nc              	      s2   g | ]*\}}t j d g| g| gdqS )r   r.  r=  )r|   xyr   r   wr   r   r     s   z8_generic_rnn.<locals>.reform_weights.<locals>.<listcomp>r   r   r   )r   r   )r   r  r   Z	intervalsZslicesr   r  r   reform_weights  s    z$_generic_rnn.<locals>.reform_weightsc                    s`   |  }dkr|\}}n,dks*dkrF fdd|D \}}t  fdd||fD S )Nr  r  r  c                    s   g | ]} |qS r   r   r|   r  r   hidden_sizereform_permutationr  r   r   r     s    zC_generic_rnn.<locals>.transform_weights_no_bias.<locals>.<listcomp>c                 3   s   | ]}t  |d gV  qdS r  ry   r|   r  r   r   r   rx    s    zB_generic_rnn.<locals>.transform_weights_no_bias.<locals>.<genexpr>)r  )layer_indexweights	weight_ih	weight_hhr   r  layer_weightsr  r  variantr   r   transform_weights_no_bias  s    

z/_generic_rnn.<locals>.transform_weights_no_biasc                    s|   |  }dkr|\}}}}n0dks.dkrN fdd|D \}}}} j d||dd}t fd	d
|||fD S )Nr  r  r  c                    s   g | ]} |qS r   r   r  r  r   r   r     s    z;_generic_rnn.<locals>.transform_weights.<locals>.<listcomp>r   r   r/  c                 3   s   | ]}t  |d gV  qdS r  ry   r  r  r   r   rx    s   z:_generic_rnn.<locals>.transform_weights.<locals>.<genexpr>)r   r  )r  r  r  r  Zbias_ihZbias_hhbias_concatr  r   r   transform_weights  s    z'_generic_rnn.<locals>.transform_weightsc                    s&   dkr| S t j | dg|g|gdS )Nr+   r   r.  r=  )r  r  r  )r   
num_layersr   r   retrieve_state  s        z$_generic_rnn.<locals>.retrieve_stater   r/  Zdirection_sbidirectional)r  hidden_size_iZactivations_s)r  r  Zlinear_before_reset_ir  )r  r  rF   r   rG   r  r   )r  )r  )r  )r   )r   )r  r  dictr  r>   r   r3   ro   r   r   
startswithr  r  r   r   r!   rH   r   r-  )*r   r  r   Zinitial_statesr  
has_biasesr  r  r  r  batch_firstbatch_sizesZonnxActivationsZvariantToOnnxActivationMapZnonlinearityw_hhZunidirectionalZprev_outputh_outsZh0Zc0c_outsZsequence_lensr  r  r  rr   r  r  r  Zstate_indicesZweight_ih_fZweight_hh_fZbias_fZweight_ih_bZweight_hh_bZbias_brG  extra_kwargsZ
activationZh_outZc_outr   )	r  r   r  r  r  r  r  r  r  r   _generic_rnn  s(    




	




 
 
  

"
"r  c
                 C   s2   t |t | }
}t| d||
|||||||	S )Nr  r   rt   r  )r   r   hidden_vweight_vr  r  r  r  r  r   hiddenr  r   r   r   
_lstm_fulll  s$    r  c
                 C   s4   t |t | }
}t| d||
||||||	|dS )Nr  r  r  )r   r   r  r  r	  r  r  r  r  r  r
  r  r   r   r   _lstm_packed  s$    r  c                 G   s.   t |d rt| f| S t| f| S d S Nr  )r   r/   r  r  r   rA   r   r   r   lstm  s    r  c                    s   t  |dg}t |} fdd|D }t |rB||||fn||f}t |rXdnd}	t d||||	dddddd\}
}}t  |dgt  |dgfS )	Nr   c                    s   g | ]}t  |d gqS r  ry   r  r  r   r   r     s     zlstm_cell.<locals>.<listcomp>TFr  r+   )r  r  r  r  r   )r   rz   rt   r  r  r-  )r   r#   r
  Zw_ihr  Zb_ihZb_hhr   r  r  r~  r  r  r   r  r   	lstm_cell  s4    
  r  c                    s^   t ddddddddd	fdd t ddddddddd	fdd fdd	}|S )
NrC   rr   rD   c
                    s&   t |}
t|  |||
||||||	S r    r  )r   r   r
  r	  r  r  r  r  r  r   r  r   r   r   	_rnn_full  s    
z"_one_hidden_rnn.<locals>._rnn_fullc
                    s(   t |}
t|  |||
|||||	|dS )Nr  r  )r   r   r  r
  r	  r  r  r  r  r  r  r  r   r   _rnn_packed  s    
z$_one_hidden_rnn.<locals>._rnn_packedc                    s.   t |d r| f| S  | f| S d S r  )r   r/   r  )r  r  r   r   r     s    z!_one_hidden_rnn.<locals>.symbolicr   )r   r   r   )r  r  r   r   _one_hidden_rnn  s    r  r  ZRNN_TANHZRNN_RELUc                 C   sX   |  d|}| j d|| j dt|ddd}t rB|  d|S t| |dd d d S d S )	Nr   r  rF   rG   r   r/  z_caffe2::Ranger  )r   rH   rI   r   r   arange)r   likerv   Z
like_shapestopr   r   r   _dim_arange  s       r  c                 C   s   |S r    r   r   r   r   r   detach  s    r  c                 C   s   |dkrt d|S )Nr   z-onnx memory_format support is not implemented)rQ   )r   r   r  r   r   r   
contiguous$  s    r  c                 C   sf   |r| j d|dddgd}| tjj s8td|  dkrTt	| |d}| j d	||dd
S )Nr   r+   r   r   r   z(Lengths must be a Tensor for ONNX exportr  Fzprim::PackPaddedr  )
r   rX   r  rH   r   Z
TensorTypegetrQ   rY   Z	_cast_Int)r   r   lengthsr   r   r   r   _pack_padded_sequence+  s    r  c                 C   s8   | j d||dd\}}|r0| j d|dddgd}||fS )Nzprim::PadPackedr   r  r   r+   r   r   r   )r   r  r  r   Zpadding_valuetotal_lengthr  r   r   r   _pad_packed_sequence<  s    r!  c                 G   sz   t |dd}|d krt jj}t |d}t |rl| jd|tjdgt j	d dd}| jd	|t j
| d
S | jd|dS )Nrr   r_   r   r  r   rN  r^   rG   RandomNormalLikedtype_iZRandomNormalshape_ir   r   r   r  r   r.   r   rH   rI   r   r   r   shapesr_   r  r   Zshape_constr   r   r   randnI  s&    
 r*  c                 G   sz   t |dd}|d krt jj}t |d}t |rl| jd|tjdgt j	d dd}| jd	|t j
| d
S | jd|dS )Nrr   r_   r   r  r   rN  r^   rG   RandomUniformLiker#  ZRandomUniformr%  r'  r(  r   r   r   rand^  s&    
 r,  c                 C   s4   t |dd}|d krt jj}| jd|t j| dS )Nrr   r_   r"  r#  r  r   r#   r_   r  r  r  r  r   r   r   
randn_likes  s      r.  c                 C   s4   t |dd}|d krt jj}| jd|t j| dS )Nrr   r_   r+  r#  r  r-  r   r   r   	rand_like~  s      r/  c                 C   s    | j d|||d}|  d||S )Nr+  )high_flow_fr=  r   )r   r   r  upperr$  r   r]  r   r   r   rrelu  s    r3  c                 C   s   |d k	rt dd |d k	r2t |s2t dd t |}|d krPt ddS | jd|ddt j| d}| d	||}| jd
|t j| dS )NZ	Bernoulliz,out parameter is not supported for bernoulliz(generator is not supported for bernoulliinput dtype not accessibler+  rE   r   )r0  r1  r$  r  rS   rU   )r   r3   r   r   r   rW   )r   r   r   r[   r_   r]  r  r   r   r   	bernoulli  s,      
r5  c                 C   s   |  d|}|  d|S )Nr   rb  r   )r   r   r]  r   r   r   log_sigmoid  s    r6  c                 C   s   |  d|S )NErfr   r   r   r   r   erf  s    r8  c                 C   s   t |}|d krt ddS |dk r.|| }|dkrR||d krR| jd||dS |dkrz||d krz| jd||d dS t | ||||S )Nrv   r
  r   r+   Flattenr/  r   )r   r   r3   r   Z_flatten_helper)r   r   Z	start_dimZend_dimrv   r   r   r   flatten  s    
r:  c                 C   s   t | | d|S )z/Emitted from `torch.nonzero(x, as_tuple=False)`ZNonZero)r}   r   r   r   r   r   r    s    r  c                 C   s   t | t| |d|dS )Nr+   )r!  )r,  r  )r   r   r!  r   r   r   nonzero_numpy  s    r;  c                 C   s   |  d|}|S )NZIsNaNr   )r   r   r  r   r   r   isnan  s    r<  c                 G   s~   t |dkr|d }d\}}n$|\}}}t|dg}t|d}t| |d}tj| |||d}t| || jdtdgdS )	Nr+   r   r=   rr   Fr   rF   rG   )	r>   r   r  r   r_  r  r   rH   r   )r   rA   r   rv   r   Z	input_sumr   r   r   _any  s    

   r=  c              	   G   sP   |  d|d }t|dkr.|  dt| |S |  dt| ||d |d S d S )Nrb   r   r+   r   )r   r>   r=  )r   rA   r   r   r   r   _all  s    r>  c                 C   s   t j| ||g|g|| gdS )Nr.  r=  )r   r   rv   r  lengthr   r   r   narrow  s        r@  c              	   C   sj   t |r<t | || jdtdgd}| jd|dddS t |d}t |d}| jd|||dS d S )	NrF   r   rG   ru  r   Frv  rr   r   r   r!   r   rH   rI   r  r   r   rv   r   Z	flattenedr   r   r   argmax  s    
  rC  c              	   C   sj   t |r<t | || jdtdgd}| jd|dddS t |d}t |d}| jd|||dS d S )	NrF   r   rG   r|  r   Frv  rr   rA  rB  r   r   r   argmin  s    
  rD  c                 C   s   |   }t|}t|r4| jd||||dS |   |krb| jd|tj|    d}| jd||t| |||dS d S )NZScatterr/  rS   rU   )rX   rY   r   r2   r.   r   rW   r   )r   r#   rv   r   srcZsrc_typer   r   r   rN    s    

rN  c                 C   s   t |}|d krt ddS t jt j| }t j| }t j|dd}|rf| jdt	j
||dd}nt j|}t| ||}t | ||||}t| ||S )Nscatter_addr4  F)Zallow_nonstaticrF   r^   rG   )r   r   r3   r   r   rW   r   rq  r   rH   r  r  Z_scatter_helperr,   )r   r#   rv   r   rE  r_   rs  Zto_addr   r   r   rF  &  s"    
 
rF  c              	   C   s*   d}|  dt| || j dt|gdS )Ng9B.?rR   rF   rG   re  )r   r#   Z_ln2r   r   r   log2;  s    rG  c                 C   s6   t |r | jdtdgdS | jdtdgdS NrF   r+   rG   r   )r   rZ   r   rH   
BoolTensorrp   r   r   r   is_floating_point@  s    
rJ  c                 C   sL   t |r@t |r*| jdtdgdS | jdtdgdS t| ||S rH  )r   r   r   rH   rI  r  r;   r   r   r   __is_F  s
    

rK  c                 C   s   t | ||S r    )rK  r;   r   r   r   __isnot_N  s    rL  c                 C   sR   | j dtddgd}|  dkr>| j d|tjd d}| j d	|||d
dS )NrF   r   r+   rG   )ByteCharr  ZShortrS   rT   rU   OneHotr   r/  )r   rH   r   rX   rY   r   rW   )r   r#   Znum_classesrJ  r   r   r   one_hotS  s      rP  c           	   	   C   s   t |drt ddS |  }| jdtddgd}t| || jdt|gd}| jd| jd	||||d
t j	| d}| dt 
| ||d g|}t j| ||gddS )Nrr   gatherzsparse_grad == TruerF   r   r+   rG   rS   rO  r/  rU   r:   r   )r   r   r3   rX   rY   r   rH   r   r   rW   rz   r_  )	r   r#   rv   r   Zsparse_gradr_   rJ  depthr<   r   r   r   rQ  ]  s    rQ  c              	   C   s<  |d kr(| j d|dd}|}t| |}nb| j d|||d}| j d||dd}|  d|}| j d|| j dt|d	dd
}| j d|dd}|  d||}	|  d|	|	}
|d krdn|}| j d|
||d}|d krd}|dkr4| j d|tjd d}| j dtj|tjdd	}|  d||}|  d||  d||}||fS )Nr   r   rt  r   r+   r   r  rF   rG   r/  r   r7   r:   rS   rV   rU   r^   rR   )r   r  rH   rI   r   rW   rm   )r   r   rv   Z
correctionr   r   Zt_meanZnum_elementsZredudced_dimsZsub_vZsqr_subZkeepdim_meanvarrg   r<   r   r   r   	_var_meano  s<    
  rT  c                 G   s    t | |f| \}}| d|S r   var_meanr   r   r   rA   rS  r~  r   r   r   std  s    rX  c                 G   s   t | |f| \}}|S r    )rV  rW  r   r   r   rS    s    rS  c                 G   s4   t |dkr t| |d |d d S t| |f| S d S )Nr+   r   )r>   rT  )r   r   rA   r   r   r   rV    s    rV  c                 G   s$   t | |f| \}}| d||fS r   rU  )r   r   rA   rS  r   r   r   r   std_mean  s    rY  c                 C   s   | j d|||dS )NZReduceLogSumExpr   r   )r   r   rv   r   r   r   r   	logsumexp  s    rZ  c           
         s  t  r jd| S dd } fdd}t|dksBt|dkrt|dkrTd }n||d }t j |d	 |d
\}}}}t  |d	g}||}t  t t ||d d dg}	 j	d|	t j
| dS t|dkst|dkrt|dkrd }n||d }t j |d	 |d |d |d\}}}}t  |d	g}t  |d	g}t  |d	g}| 	d 	d|||}t  t t |d d d dg}	 	d 	d|	||}	 j	d|	t j
| dS t|dkr||d }t j |d	 |d |d\}}}}t  |d	g}t  |d	g}| 	d||} 	dt  t t ||f|dd   dg|}	 j	d|	t j
| dS tdtt| d d S )Nr  c                 S   s   t | d} | S )Nrr   )r   r   r^   r   r   r   _get_arange_dtype  s    z!arange.<locals>._get_arange_dtypec                    s,   t | r( jd d| t jd d} | S )NrS   rQ  r  rU   )r   rZ   r   r   )range_tensorr  r   r   _float_step_convert  s    

z#arange.<locals>._float_step_convertr   r  r+   r   )r  r_   rS   rU   r  r  r  )r  r  rB  r_   rR   r7   r(   r:   rN  )r  r  r_   z&Unknown aten::arange signature taking z arguments.)r  )r   r   r   r>   Z_arange_cast_helperrz   r-  r  r   r   r   r   r;  )
r   rA   r[  r]  r_   r  r  rB  r\  Zarange_tensorr   r  r   r    s    	                     r  c           
      C   sT   t | |d }t| t| ||t| || jdtjdtjdd}	t| t	| ||	|S )NrF   r+   r^   rG   )
r   Z_arange_helperrB   r6   r   rH   rI   rd   r,   r<   )
r   r  r  Zstepsr_   r  r  r  r\  rB  r   r   r   linspace  s    
 r^  c                 C   s   |S r    r   rp   r   r   r   lift  s    r_  c              	   C   s0   t | |d}t|}| d|t| |||S )NFr  )Z
_cast_Boolr   r2   r   r   )r   r#   maskrJ   r   r   r   masked_fill  s    
ra  c                    s*  t  rjd||ddS t |r0t |}n|g}fddfdd|D }t|dkrxt j|d	|d	 d
dS dd t|D  t d	kr|S t dkrt| d	 | d	  S t 	|}|d krt
dtdttj d d d  t }t|fddt|D jd|  fddt|D  d}jd||d}| d  } d  }t|d ddD ]@}d| |  |}	d||	}d| |  }qt|d	|}t|}
 tt d	  d d krjdtdgdg fddt|D  }jd$|d d	i}t ||}ttd d	 d d	g tt d	 d || d  }jd||d}fd!dt d	 D |
g  fd"dt d	 |D  }jd%|d d	i}n,jd|
f fd#dt|D d d	i}t ||S d S )&Nr   r   )r  c                    s`   t | s\|   dks*|   dkr\tjdk r<tdtd t 	 t
 | dg} | S )NrM  r  r)   z?Exporting masked indices are only supported after ONNX opset 9.zExporting aten::index operator with indices of type Byte. Only 1-D indices are supported. In any other case, this will produce an incorrect ONNX graph.r+   )r   r   rX   rY   r	   rE  rQ   r  r  r-  r  )r   r  r   r   try_mask_to_index!  s    

z index.<locals>.try_mask_to_indexc                    s   g | ]} |qS r   r   )r|   r  )rb  r   r   r   1  s     zindex.<locals>.<listcomp>r+   r   F)Zapply_reshapec                 S   s   g | ]\}}t |s|qS r   )r   r   )r|   rr   r  r   r   r   r   I  s    
 zUnsupported aten::index operator of advanced indexing on tensor of unknown rank, try turning on shape and type propagate during export: torch.onnx._export(..., propagate=True).z=Exporting aten::index operator of advanced indexing in opset z8 is achieved by combination of multiple ONNX operators, z2including Reshape, Transpose, Concat, and Gather. zVIf indices include negative values, the exported graph will produce incorrect results.c              
      s0   g | ](} j d  j dt|gdddqS )r  rF   rG   r   r/  )r   rH   r   )r|   rv   )r   shape_tensorr   r   r   f  s   r   c                    s   g | ]}| kr|qS r   r   rf  )adv_idx_indicesr   r   r   t  s      r   r9  r/  r   r   r:   r(   rF   rG   c                    s   g | ]}| kr| qS r   r   rf  rd  dim_tensor_listr   r   r     s     r   r   c                    s   g | ]} | qS r   r   rf  )rf  r   r   r     s     c                    s   g | ]}| kr| qS r   r   rf  re  r   r   r     s   c                    s   g | ]}| kr| qS r   r   rf  re  r   r   r     s   )r   )r   )r   r   r   r   rt   r>   rG  rF  rH  r   r   r  r  r;  r	   rE  r   r   r   r   rH   r   r!   )r   r#   r   r  r  Zadv_idx_countZcum_adv_indexZ
multiplierrr   Z	adv_indexZcum_adv_index_shape_tensorZfolded_adv_idx_shape_listZfolded_adv_idx_shapeZadv_idx_permuteZfinal_shape_listZfinal_shaper   )rd  rf  r   rc  rb  r   r     s    
       



  

 	r   c                 C   s   d }|d krzt |r<t | |dg}| jdtdgd}t |}|d krZt ddS |dkrpt |d}qd	dg}n8t	|dkrt |r| jdtdgd}t |d}|rt
| |||||S t| |||||S )
Nr   rF   r   rG   rv   (Input rank must be known at export time.r+   rD   r   )r   r   r!   r   rH   r   r   r3   r  r>   linalg_vector_normlinalg_matrix_norm)r   r#   ordrv   r   r_   	ord_valueself_dimr   r   r   linalg_norm  s*    

 

rm  c                 C   s   |d krt | |dg}d }|tjkrB| jd| d|||d}n|tj krj| jd| d|||d}n||dkrt ddd	d
S | jdt|gd}t j| | d| d||||d}| d|| d| jdtdgd|}|S )Nr   r]  r`  r   r{  r   rh  r)   r*   zord=0 not supportedrF   rG   r  rR   r+   )	r   r!   ri  infr   r0   rH   r  r_  )r   r#   rj  rv   r   r_   resultZord_opr   r   r   rh    s6    
      rh  c              	   C   s  t |d}|dkr"t| |||S |dkr6t ddS t |d}|d krXt| |||S |dksh|dkrtt dd	S t |}|d krt dd
S |d dk r|d  |7  < |d dk r|d  |7  < |tjks|tj k r|d |d  |d< |d< |d |d kr$|s$|d  d8  < t j| | d||d g|d}|dkrzt	| || jdt
|d gd|d\}	}
n*t| || jdt
|d gd|d\}	}
|	S d S )NrL   ZfroZnuczlinalg.matrix_normzord==nucrD   r   r  zord==2rg  r   r+   r`  r   rF   rG   )rx  r   )r   r  frobenius_normr3   r   ri  rn  r_  r   r  rH   r   ro  )r   r#   rj  rv   r   r_   rk  rl  r   ro  r  r   r   r   ri    sX    
  
  

ri  r   c                 C   s   t | |||S r    )r\  )r   r   r&   rv   r   r   r   linalg_cross%  s    rq  c                 C   s,   |  d||}tj| |||d}|  d|S )Nr:   r   r   )r   r   r_  )r   r#   rv   r   ZsqrZsumsqrr   r   r   rp  *  s    rp  c                 C   sX   |d k	rt |st dd |s6|dkr6t dd t| |}| jd|t jd |dS )NZMultinomialz*generator is not supported for multinomialr+   zGreplacement=False when num_samples > 1 is not supported for multinomialrT   )r$  Zsample_size_i)r   r   r3   rc  r   rW   )r   r   Znum_samplesreplacementr   Z	log_inputr   r   r   multinomial1  s"     
rs  c           
      C   s`   |   }t| ||}t| || jd|tj| d}t| || jd|tj| d}	t| ||	S r  )rX   rY   r   r<   r   r   rW   r,   )
r   r#   Zbatch1Zbatch2r   r5   r_   Z	batch_mulZmul_aZmul_br   r   r   baddbmmF  s      rt  )indexingc                    s,  |d krd}n|dkr$t d| |dkrF|d |d  |d< |d<  fddt|D } fd	d|D } jd|ddi}g }t|D ]h\}} jdtjdtjddgt| }	|| |	|< t	 | jd|	ddi}
|
 d|
| q|dkr|d |d  |d< |d<  jd| S )Nij>   rv  xyzUnsupported indexing: rw  r+   r   c                    s,   g | ]$}t  | jd tdgdqS )rF   r   rG   )r   r!   r   rH   r   r{   r  r   r   r   \  s     zmeshgrid.<locals>.<listcomp>c                    s   g | ]}  d |qS )r   r   r{   r  r   r   r   b  s     r   r   rF   r^   rG   r   prim::ListConstruct)r   )r   )rx  )
ValueErrorr   rt   r   rF  rH   r   rd   r>   r   r   )r   ru   ru  rw   Ztensors_shapeZ	out_shaper[   rr   r}   r&  Z
t_reshapedr   r  r   meshgridT  s,    

rz  c                 C   s(   t | ||}| d||}| d||S )Nr:   r7   )rO   r   )r   r   r&   rB   Zquor   r   r   	remainderq  s    r{  )r#   approximatec                 C   s&  |dkrt dt j }d}tj|tjd}tj|tjd}tjdtjd}tjdtjd}t| |t| ||}	t| |t| |t| ||	}
t| |t| |t| || d|
S d}| d	| d
|tj|tjd}t| || jdtjdtjdd}t| t| ||| jdtjdtjddS d S )Nr   r   gHm?r^   rE   g      ?r   g;f?r7  rR   rF   r+   rG   )	ri  r   pirH   rI   rn   r<   r,   r   )r   r#   r|  ZkBetaZkKappar   kapparg   ZhalfZ	self_cubeinnerZ_sqrt2r8  Zerf_plusoner   r   r   geluw  s,    $"  
r  c              
   C   s  t  r | jd||||||dS t |d}|d k	rD|| dksDtt |}|d krbt ddS d|dg}	t | || jdt	
|	d}
| jdt	d	g| d
|   d d}| jdt	dg| d
|   d d}| jd|
|||d}t | || d|}|d ks,|  r\t	d	gd
|   d }| jd|d}|d kst|  rt	dgd
|   d }| jd|d}ttd|d }t| t| |t | ||t | ||S )N
group_norm)Znum_groups_ir)  Zcudnn_enabled_ir+   r   zunknown input rankr   rF   rG   rE   r/  r   r   r0  r1  r   )r   r   r   r  ro   r   r3   r!   r   rH   r   rI   rX   rY   r   
mustBeNoner   r   r,   r<   rz   )r   r   Z
num_groupsr  r  r&  r  r4  Z
input_rankr   r;  r9  r:  Znorm_reshapedr  r5  r6  r   r   r   r   r    st    


      r  c                 C   s   t |}|d k	rttt|}|d k	rH|dk r6||7 }|dkrH|| t| |d|d}| d||}| d||S t  r| jd|||dS t	dd S )	Nr   r   r+   rR   r:   _weight_normr   zDUnsupported: ONNX export of _weight_norm for tensor of unknown rank.)
r   r   r   r   remover  r   r   r   rQ   )r   r	  Zweight_grv   r  r   Znorm_vrB   r   r   r   r    s     

r  c                 C   s   |  d|}|  d|S )zFImplement the dim functionality available for a pytorch tensor in ONNXr   Sizer   r"   r   r   r   rv     s    rv   c                 C   s    t | || jdtdgd|S rS  )r1  r   rH   rI   )r   r#   rr   r   r   r   
__getitem_  s    r  c                 C   s   |S r    r   rp   r   r   r   r    s    r  c              
   C   sD   t | || jdtjdgtjdd}t| |d|}t| ||}|S )NrF   r   r^   rG   r   )r   r!   r   rH   rI   rd   rH  r'   )r   r#   r   Zself_flattenedr[   r   r   r   take  s      r  c                 C   s&   t | ||}t| |}t| ||}|S r    )r6   rd  r<   )r   r   targetdiff_Zexp_r  r   r   r   _kl_div_log_target_impl  s    
r  c           	      C   sZ   t | |}t| ||}t| ||}t| |}t| || jdtdd}t| |||}|S rS  )	rc  r6   r<   r  r  r   rH   rI   r   )	r   r   r  Zlog_r  Z
output_posZzeros_Zmask_r  r   r   r   _kl_div_non_log_target_impl  s    

r  c                 C   sh   |rt | ||}nt| ||}|dkr*|S |dkrB| jd|ddS |dkrZtj| |ddS tdS d S )Nr   r+   r   rt  r   z4kl_div with reduction other than none, mean, or sum.)r  r  r   r   r_  r   )r   r   r  Z	reductionZ
log_targetr  r   r   r   kl_div  s    r  c                 C   s  t |d}t|}t | || jdtjdgtjdd}t |stjdgtj	d}t
t||D ]6\}\}	}
dg| }d||< |t|	||
  }qd|r|| }| d|| jd|dS d }t
|D ]\}}
dg| }d||< t| || jdtdgd| jdt|d}	t | t| |	d	d d d | jdt|d}| d
|| jdt|
gd}|d krr|}q| d||}q|r| d|| dt|g}| d||S d S )Nr   rF   r   r^   rG   r   r+   r  r  r:   r(   )r   r   r>   r!   r   rH   rI   rd   r.   longrF  r  r  r  r1  )r   r#   rs  stridesoffsetr  Zself_1dindrr   r   rl  Zr_sizeZtmp_indr   r   r   
as_strided*  sT      


  
r  c              	   C   s   |  d||  d||S )Nr(   r:   r   )r   r   r  rB  r   r   r   __derive_indexW  s    r  c                 C   s8   |  d||}|  dt| ||}| j d|tjd dS )Nr7   rQ  rS   rT   rU   )r   r?   r   rW   )r   lor?  rB  r6   rB   r   r   r   __range_lengthc  s    r  c                 C   s   t |}t| |}|dkrp|  sp| jdtjdtjdd}| jdtjdtjdd}t	| |||||}n$t
| ||}|  st| ||}|S )Nr   rF   r+   r^   rG   )r   r   r}   r   r  r   rH   rI   rd   r   r   r,   )r   r   r  r  r  r5   r   r  r   r   r   r  i  s    

c           
   	   C   s   |d kr,t  }|r|js t j}tj|}t| |dd d d }| jd|tj	d d}	t
| | jdt jtjt jdd|	}	|dkrt| || jdt jd	t jdd}t| |	|}	| jdt| t| |	tj| d}	|	S )
Nr  rS   rV   rU   rF   r^   rG   Fr+   )rH   rl   rJ  rm   r   r   r   r  r   rW   r<   rI   ri  r}  r6   r  rB   r2  r   r   )
r   Zwindow_lengthZperiodicr_   r  r  r  r  Zn_arrayr  r   r   r   hann_windowx  s2    
    r  c                 C   s   t | ||S r    r   )r   r#   Zvecr   r   r   mv  s    r  c                 C   s   t | ||S r    r  r;   r   r   r   dot  s    r  c                 C   s>   |   }|d krtjj}ntjtj| }t| |||S r    )	rX   rY   r   r   r  r   r   rW   r  )r   r#   rJ   r_   r   r   r   fill  s    
r  c                    s  t d |r.tt|dkr.tddS t d  d krJtdt|}t|}|d ksn|d krvtd||kr|| }t	|D ]}	t
| |t|g}qt| }
t| }|
d k	r|d k	r|
|krtdtt	|}d	d
 t	|D } fdd
t	|D }tj| ||||d}t| ||}t	 D ]}	t
| |dg}q>t	|  d D ]}	t
| |t|g}qft| | t| |||S )NzyWarning: ONNX export does not support duplicated values in 'index' field, this will cause the ONNX model to be incorrect.r+   	index_addr-   rr   zXONNX export does NOT support exporting 'index_add_()' function with unknown 'dim' value.z~ONNX export does NOT support exporting 'index_add_()' function while the rank of self tensor or tensor to be added is unknown.zoONNX export does NOT support exporting 'index_add_()' function with duplicated values in 'index' parameter yet.c                 S   s   g | ]}d qS r  r   rf  r   r   r   r     s     zindex_add.<locals>.<listcomp>c                    s   g | ]}| krt jnd qS rm  )sysmaxsizerf  rv   r   r   r     s     r.  r   )r  r  r   r1   r2   r3   r   r   r   r   rz   r  r   r0  r   rF  )r   r#   rv   r   r&   r5   Zself_dim_rankZother_dim_rankdeltarr   Zother_dim_sizeZself_dim_sizeZnew_shape_axesZnew_shape_startsZnew_shape_endsrU  r   r  r   r    sf    

  
      
r  c                 C   s   t |t |kst|}tt |D ]~}g }tj| ||| g||  gtjgd}|| tj| ||| gdg||  gd}|| | jd|d|| i}q$|S )Nr.  r   r   r   )r   )	r>   ro   r   r   r0  r  r  r   r   )r   r#   Zshiftsr  ro  rr   r)  r   r   r   r   roll  s,       
 
    

r  c                 C   sp   t ||}t| |dg|g}t| |dg|g}t| |dg|g}t| |dg|g}t| t| ||t| ||S )Nr   r+   )r   Z_get_dim_for_crossr  r6   r<   )r   r   r&   rv   Zroll_x_1Zroll_y_1Zroll_x_2Zroll_y_2r   r   r   r\    s    r\  r*  #use_mm_for_euclid_dist_if_necessaryc                 C   sF   t |}t | ||d g}t | ||d g}t| |||dddS )Nr+   r   gư>F)r&  r   )r   r   rz   r^  )r   rZ  r[  r]  Zcompute_moder  Zbroadcasted_x1Zbroadcasted_x2r   r   r   cdist  s    
     r  c                    sP   t |}t |d |D ]}t |q fdd|D } jd| S )Nr   c                    s   g | ]}t  |qS r   )r   r{   r   Zt_with_final_shaper   r   r   2  s     z%broadcast_tensors.<locals>.<listcomp>rx  )rx  )r   rt   r  r,   r   )r   r#   Zall_tensorsr}   Zt_listr   r  r   broadcast_tensors)  s    
r  c                   @   s  e Zd ZdZedd Zedd Zedd Zedd	 Zed'ddZ	edd Z
edd Zeeeej  dddZedd Zedd Zedd Zedd Zedd Zeejjddd Zeejjdd!d"Zeejjdd#d$Zeejjdd%d&Zd
S )(PrimZprimc                 C   s\   t ||}|d kr t ddS |g||  }|| }|rD|| | jd|||t|dS )Nzprim::ConstantSplitr  r  r  )r   r  r3   r   r   r>   )r   r#   r"  rv   r   r#  r$  r   r   r   ConstantSplit9  s     
zPrim.ConstantSplitc                 C   s@   t ||}|d kr t ddS || d | }t| |||S )Nzprim::ConstantChunkr  r+   )r   r  r3   r  r  )r   r#   r   rv   r<  r"  r   r   r   ConstantChunkJ  s     zPrim.ConstantChunkc                 C   s   |  d|S r   r   rp   r   r   r   r   T  s    z
Prim.shapec                 C   s   t | d||ddS )Nrq  ri  rK  rL  r;   r   r   r   r  X  s    zPrim.maxNc                 C   sB   |s6t |r,t| || jdtdgd}t| |S t| ||S rS  )r   r   r   r   rH   rI   ro  r;   r   r   r   ro  \  s
    

zPrim.minc                 C   s   |S r    r   rp   r   r   r   r  d  s    z	Prim.datac                 O   s   d S r    r   r   rG  r   r   r   r   ListConstructh  s    zPrim.ListConstruct)returnc                 O   s2   t |dkr.|d   dkr.t|d S d S )Nr+   r   rx  )r>   r   r   r   rt   r  r   r   r   
ListUnpackl  s     zPrim.ListUnpackc                 O   s   d S r    r   r  r   r   r   TupleConstructu  s    zPrim.TupleConstructc                 O   s   d S r    r   r  r   r   r   Uninitializedy  s    zPrim.Uninitializedc                 C   s   |S r    r   rp   r   r   r   unchecked_cast  s    zPrim.unchecked_castc                 C   s<   t |}|d krd}t jt j| }| jdt|dS )NrV   rF   rG   )r   r   r   r   rW   r   rH   rI   r   r   r   r   r_     s    
z
Prim.dtypec                 C   s$   t |d}|dkr t ddS |S )Nrr   r+   zprim::tolistzdim_val > 1)r   r   r3   )r   r   Zdim_valZelem_ty_valrv   r   r   r   tolist  s    zPrim.tolistctxc                 O   s*   | j }|   dkrd S tddS )Nr  r  z#output type is not `DeviceObjType`.)cur_noder  rX   r   r   r3   )r  r   rG  r   r   r   r   r   r    s     zPrim.devicec                 O   s$  | j }| j}| j}tj}tj}|jd|d| i}	| dkrN|	d  n|	 }
|	 D ]}|

 }t| D ]l\}}|dkr|t|k r|||   |dkrv|d t|k rvt| tjsv|||d    qvtj||||d q^tj|
|}	tjr tj|
|| |	S )NLoopr  r+   r   F)r  )r  envparams_dictr	   r  rE  r   outputsSizer   blocksaddBlockrF  rG  r>   r   rX   r   r   r   rH   _jit_pass_onnx_block%_jit_pass_fixup_onnx_controlflow_nodeonnx_shape_inference(_jit_pass_onnx_node_shape_type_inference)r  r   rG  attrsr   r  r  r  opset_versionnew_op_outputsnew_noder  	new_blockrr   Zb_inr   r   r   r    sL           z	Prim.Loopc                 O   s  | j }| j}| j}| j}tj}tj}	|d   dk}
|
r|d  d 	 }t
|trdt|nt|}|rtdnd}t| | }tj||||d}t| }t| }g }tt|D ]8}|| |krtd|| |||  }|| q|S |jd
|d| i}| dkr4|d  n| }| D ]"}| }tj||||d	 qDtj||	}tjrtj|||	 |S d S )Nr   r   rJ   r+   Tz+The sub block ATen output {} is not in env.Ifr  F)r  )r  
onnx_blockr  r  r	   r  rE  r   r   r  r   r   allr  r  rH   r   r  r  r   r>   rQ   r   r   r   r  r  r  r  r  )r  r   rG  r  r   blockr  r  r  r  Z	static_ifZ
input_flagr  Z	block_idxZ	current_bZif_output_listZcurrent_b_listZfinal_b_listr  Zonnx_br  r  r  r  r   r   r   r    st    
   zPrim.Ifc                 O   s   | j }| rd S t|  tjr*d S |ddkrJ|jd|d dS |ddkrj|jd|d dS |  	tj
 s|  	tj
 r|jdt|d dS td|dd S )NrJ   r}   rF   rG   rL   )Zvalue_sz9Unsupported prim::Constant kind: `{}`. Send a bug report.)r  r  r   r  rX   r   r  ZkindOfr   r  r  r  ZofFloatsrH   rI   rQ   r   )r  r   rG  r  r   r   r   r   rF   .  s(    zPrim.Constant)N)r   
__module____qualname__domainstaticmethodr  r  r   r  ro  r  r  r   r   r   r  r  r  r  r  r_   r  rH   onnxSymbolicContextr  r  r  rF   r   r   r   r   r  6  sF   

	








	
-Xr  c                   @   s&   e Zd ZdZeejjdddZdS )Onnxr  r  c                 O   s"   | j }| j}| j}tj|||S r    )r  r  r  rH   r   Z'_jit_onnx_convert_pattern_from_subblock)r  r   rG  r  r   r  r  r   r   r   PlaceholderO  s    zOnnx.PlaceholderN)	r   r  r  r  r  rH   r  r  r  r   r   r   r   r  I  s   r  )N)N)N)rE   )T)T)N)N)N)N)N)N)r   N)N)F)N)N)NNN)N)N)FF)NN)NN)FN)NNNFN)F)NNF)NN)F)NNNFN)F)F)NNNFN)F)F)NNNFN)F)N)N)NN)NN)NNFN)NNFN)NN)N)F)r   )NF)FN)N)r   )N)TNNNNF)N)N)r*  r  (`  __doc__r   ri  r  r  typingr   r   r   r   rH   Ztorch._C._onnxr   Z_onnxrT  Ztorch.nn.modules.utilsZ
torch.onnxr   r   Ztorch.onnx._globalsr	   r   r   r   r   r'   r,   r6   r8   r<   rB   r   rK   r@   rP   rO   ri   rj   r?   rq   rx   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r   r   r   r   r   r   r   r   r   r   r   r   r   r}   r   r   r  r  r   r  r  r  r  r  r%  r)  r'  r+  r,  r1  r2  r8  r>  r?  rA  rH  rM  rP  rR  rM   rT  rU  rY  r\  rb  re  rt  r  nnmodulesutilsZ_singler  Z_pairr  Z_tripler  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  Zreflection_pad1dZreflection_pad2dZreflection_pad3dZreplication_pad1dZreplication_pad2dZreplication_pad3dr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r  r  r  r(  r.  r<  rE  rF  rH  rI  rL  rR  rW  rX  rY  r^  r_  ra  rc  rd  rf  r+  rp  rm  rO  r  rz  ro  r}  r  r  r  rd  r  r  r  r  r  Zdropout_Zfeature_dropout_Zalpha_dropout_Zfeature_alpha_dropout_r  r  r  r  rW   itemsr  rC   r   r   partialZ_cast_func_templater  r  r  r  r  rI   r  r  r  r  r   r   r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r   r3  r  r  r  r  r  r  r  r  r  ZgruZrnn_tanhZrnn_relur  r  r  r  r!  r*  r,  r.  r/  r3  r5  r6  r8  r:  r  r;  r<  r=  r>  r@  rC  rD  rN  rF  rG  rJ  rK  rL  rP  rQ  rT  rX  rS  rV  rY  rZ  r  r^  r_  ra  r   rm  rh  ri  rq  rp  rs  rt  r;  rz  r{  r  r  r  r  rv   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r\  r  r  r  r  r   r   r   r   <module>   sL  "
 




4

$


	

!



>1


	

	<(: 
   
   
  


//  
  
  



(					




            @6!Z







        
        
        
    
G


F]
D
L  aB

            

"
Q
 "4
"
&A	
,     $
B  