U d=@sddlZddlmZmZddlZddlZddlmZGdddejjZ GdddejjZ Gdd d ejjZ Gd d d ejjZ dS) N)OptionalTuple)TensorcseZdZdZejjZdeee ddfdd Z de e e e e fe e e fddd Zdee e e e fd d d ZddZedddZeddZZS)LSTMCellaA quantizable long short-term memory (LSTM) cell. For the description and the argument types, please, refer to :class:`~torch.nn.LSTMCell` Examples:: >>> import torch.nn.quantizable as nnqa >>> rnn = nnqa.LSTMCell(10, 20) >>> input = torch.randn(3, 10) >>> hx = torch.randn(3, 20) >>> cx = torch.randn(3, 20) >>> output = [] >>> for i in range(6): hx, cx = rnn(input[i], (hx, cx)) output.append(hx) TN input_dim hidden_dimbiasreturncs||d}t||_||_||_tjj|d|fd|i||_tjj|d|fd|i||_ tjj |_ tjj |_ tjj |_tjj |_tjj |_dS)Ndevicedtyper )super__init__ input_size hidden_sizer torchnnZLinearigateshgatesZ quantizedZFloatFunctionalgatesfgate_cx igate_cgatefgate_cx_igate_cgateogate_cyselfrrr r r factory_kwargs __class__D/tmp/pip-unpacked-wheel-ua33x9lu/torch/nn/quantizable/modules/rnn.pyr s    zLSTMCell.__init__)xhiddenr cCs|dks |ddks |ddkr4||jd|j}|\}}||}||}|j||}|dd\}} } } t |}t | } t | } t | } |j | |} |j || } |j| | }|}t |}|j | |}||fS)Nrr)initialize_hiddenshape is_quantizedrrraddchunkrZsigmoidtanhrmulrrr)rr#r$hxcxrrrZ input_gateZ forget_gateZ cell_gateZout_gaterrrcyZtanh_cyZhyr!r!r"forward2s$        zLSTMCell.forwardF) batch_sizer(r cCsVt||jft||jf}}|rNtj|ddtjd}tj|ddtjd}||fS)N?rZscaleZ zero_pointr )rzerosrquantize_per_tensorZquint8)rr1r(hcr!r!r"r&Ks "zLSTMCell.initialize_hiddencCsdS)NZQuantizableLSTMCellr!rr!r!r" _get_nameRszLSTMCell._get_namecCs|dk|dkkst|jd}|jd}||||dk d}tj||j_|dk rbtj||j_tj||j_|dk rtj||j_|S)zUses the weights and biases to create a new LSTM cell. Args: wi, wh: Weights for the input and hidden layers bi, bh: Biases for the input and hidden layers Nr%)rrr ) AssertionErrorr'rr ParameterrZweightr r)clswiwhbibhrrcellr!r!r" from_paramsUs  zLSTMCell.from_paramscCs\t||jkstt|ds$td||j|j|j|j}|j |_ |j |j _ |j |j _ |S)Nqconfigz$The float module must have 'qconfig') type _FLOAT_MODULEr:hasattrrBZ weight_ihZ weight_hhZbias_ihZbias_hhrCrr)r<otherobservedr!r!r" from_floatjs   zLSTMCell.from_float)TNN)N)F)NN)__name__ __module__ __qualname____doc__rrrrEintboolrrrrr0r&r9 classmethodrBrI __classcell__r!r!rr"r s( rcsXeZdZdZd eeeddfdd Zd eee eefddd Z e d d Z Z S)_LSTMSingleLayerzA single one-directional LSTM layer. The difference between a layer and a cell is that the layer can process a sequence, while the cell only expects an instantaneous value. TNrcs0||d}tt||fd|i||_dSNr r )rrrrArrr!r"r|s  z_LSTMSingleLayer.__init__r#r$cCs<g}|D]}|||}||dqt|d}||fS)Nr)rAappendrstack)rr#r$resultZxxZ result_tensorr!r!r"r0s   z_LSTMSingleLayer.forwardcOs(tj||}||j|j|j}||_|S)N)rrBrrr rA)r<argskwargsrAlayerr!r!r"rBs z_LSTMSingleLayer.from_params)TNN)N)rJrKrLrMrNrOrrrrr0rPrBrQr!r!rr"rRvsrRcs^eZdZdZdeeeeeddfdd Zdeee eefdd d Z e dd d Z Z S) _LSTMLayerz#A single bi-directional LSTM layer.TFN)rrr batch_first bidirectionalr c sZ||d}t||_||_t||fd|i||_|jrVt||fd|i||_dSrS)rrr\r]rRlayer_fwlayer_bw) rrrr r\r]r r rrr!r"rs  z_LSTMLayer.__init__rTcCs|jr|dd}|dkr$d\}}n|\}}|jrv|dkr@d}n|d}|d}|dkr^d}n|d}|d}||f}|dkr|dkrd}ntj|tj|f}|||\} }t|dr|jr|d} | | |\} }| d} t | | g| d} |dkr"|dkr"d} d}nh|dkr>|d} |d}nL|dkrZ|d} |d}n0t |d|dgd} t |d|dgd}n| } tj|\} }|jr| dd| | |ffS)Nrr%)NNr_)r\ transposer]rjit_unwrap_optionalr^rFZflipr_catZdimrVZ transpose_)rr#r$Zhx_fwZcx_fwZhx_bwZcx_bwZ hidden_bwZ hidden_fwZ result_fwZ x_reversedZ result_bwrWr6r7r!r!r"r0sP         z_LSTMLayer.forwardrcKs:t|ds|dk st|d|j}|d|j}|d|j}|d|j}|d|j} |||||| } t|d|| _ t|d|} t|d |} t|d |d} t|d |d}t | | | || _ |jr6t|d|d } t|d |d } t|d |d d} t|d |d d}t | | | || _ | S) z There is no FP equivalent of this class. This function is here just to mimic the behavior of the `prepare` within the `torch.ao.quantization` flow. rCNrrr r\r]Z weight_ih_lZ weight_hh_lZ bias_ih_lZ bias_hh_lZ_reverse)rFr:getrrr r\r]getattrrCrRrBr^r_)r<rGZ layer_idxrCrYrrr r\r]rZr=r>r?r@r!r!r"rIs(z_LSTMLayer.from_float)TFFNN)N)rN)rJrKrLrMrNrOrrrrr0rPrIrQr!r!rr"r[s 4r[c s~eZdZdZejjZdeeee e e e ddfdd Z de e ee e fd d d Zd dZedddZeddZZS)LSTMa7A quantizable long short-term memory (LSTM). For the description and the argument types, please, refer to :class:`~torch.nn.LSTM` Attributes: layers : instances of the `_LSTMLayer` .. note:: To access the weights and biases, you need to access them per layer. See examples below. Examples:: >>> import torch.nn.quantizable as nnqa >>> rnn = nnqa.LSTM(10, 20, 2) >>> input = torch.randn(5, 3, 10) >>> h0 = torch.randn(2, 3, 20) >>> c0 = torch.randn(2, 3, 20) >>> output, (hn, cn) = rnn(input, (h0, c0)) >>> # To get the weights: >>> print(rnn.layers[0].weight_ih) tensor([[...]]) >>> print(rnn.layers[0].weight_hh) AssertionError: There is no reverse path in the non-bidirectional layer r%TFN)rr num_layersr r\dropoutr]r c s$|| d} t||_||_||_||_||_t||_||_ d|_ |rPdnd} t |t j rd|krtdkrnn t |trtd|dkrtd|dkrtd||t|j|j|jfd|j d | g} td|D],} | t|j|j|jfd|j d | qtj| |_dS) Nr Fr%rzbdropout should be a number in range [0, 1] representing the probability of an element being zeroedz|dropout option for quantizable LSTM is ignored. If you are training, please, use nn.LSTM version followed by `prepare` step.zdropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout={} and num_layers={})r\r])rrrrrhr r\floatrir]Ztraining isinstancenumbersNumberrO ValueErrorwarningswarnformatr[rangerUrrZ ModuleListlayers)rrrrhr r\rir]r r rnum_directionsrtrZrr!r"rsP    $   z LSTM.__init__rTc s|jr|dd}|d}|jr&dnd}|dkrtj|||jtj|jd d|j rptj dd|j dfddt |jD}nvtj|}t|dtr|d|j|||jd|d|j|||jdfd dt |jD}n|}t|jD] \}}||||\}||<q g} g} t |jD]*}| ||d| ||dq>t| } t| } | jd | jd d} | jd | jd d} |jr|dd}|| | ffS)Nrr%rj)r r r2r3csg|] }fqSr!r!).0_)r4r!r" Hsz LSTM.forward..cs(g|] }|d|dfqS)r)squeeze_)rvidx)r.r-r!r"rxRs)r{)r{)r\r`sizer]rr4rrkr ryr(r5r rsrhrarbrlrZreshapeZunbind enumeratertrUrVr') rr#r$Zmax_batch_sizeruZhxcxZhidden_non_optrzrZZhx_listZcx_listZ hx_tensorZ cx_tensorr!)r.r-r4r"r0:s^       z LSTM.forwardcCsdS)NZQuantizableLSTMr!r8r!r!r"r9kszLSTM._get_namecCst||jstt|ds"|s"t||j|j|j|j|j|j |j }t |d||_ t |jD]}tj|||dd|j|<q\|tjjj|dd}|S)NrCF)r\T)inplace)rlrEr:rFrrrhr r\rir]rerCrsr[rIrtevalrao quantizationprepare)r<rGrCrHrzr!r!r"rIns  zLSTM.from_floatcCstjjj|dddS)NFT)rZremove_qconfig)rrrconvert)r<rGr!r!r" from_observed}s zLSTM.from_observed)r%TFrgFNN)N)N)rJrKrLrMrrrfrErNrOrkrrrrr0r9rPrIrrQr!r!rr"rfs.*1 rf) rmtypingrrrprrrModulerrRr[rfr!r!r!r"s ib