U d3 @s"ddlmZmZmZmZmZmZmZddlZddl Z ddl Z ddl m Z ddl mZmZmZddlmZmZmZmZmZddlmZmZddlmZmZd d lmZmZm Z m!Z!dd l"m#Z#m$Z$m%Z%m&Z&dd l'm(Z(d d l)m*Z*m+Z+m,Z,d dl-m.Z.m/Z/d dlm0Z0m1Z1m2Z2m3Z3m4Z4m5Z5ddl6m7Z7m8Z8d dl9m9Z9ej:j;eeeeee)AnyDictListOptionalSetCallableTupleN) GraphModule)GraphNodeArgument)"activation_is_statically_quantizedweight_is_quantizedget_qparam_dict _parent_nameget_swapped_custom_module_class) QConfigAnyqconfig_equals)convert_dict_to_ordered_dictupdate_qconfig_for_qat)generate_qconfig_map$compare_prepare_convert_qconfig_dictupdate_qconfig_for_fusion%is_qconfig_supported_by_dtype_configs)-get_root_module_to_quantized_reference_moduleget_pattern_to_dtype_configsget_fused_module_classesget_qat_module_classes)get_native_backend_config_dict)QuantizedGraphModuleis_observed_moduleis_observed_standalone_module)update_obs_for_equalizationconvert_eq_obs)get_custom_module_class_keysget_quantize_node_infocreate_getattr_from_valuecollect_producer_nodes graph_module_from_producer_nodesWEIGHT_INDEX_DICT)_remove_qconfigis_activation_post_process)lower_to_fbgemm)observedreturncCs,t|std|j}|j}|j}|||fS)Nz-incoming model must be produced by prepare_fx)r"AssertionErrorZ_prepare_custom_config_dictZ_node_name_to_scopeZ_observed_node_names)r/prepare_custom_config_dictnode_name_to_scopeobserved_node_namesr5D/tmp/pip-unpacked-wheel-ua33x9lu/torch/ao/quantization/fx/convert.py restore_state>s r7)node qconfig_mapr0cCs"t|to |j|ko ||jdkS)z] Check if a node has a qconfig of None, i.e. user requested to not quantize the node N) isinstancer name)r8r9r5r5r6has_none_qconfigKsr<cCsn|jjD]`}|jdks|jtkr"qt|jD]:\}}|t|jkrDq,t|}|dkrVq,t||}|q,qdS)z Extract the subgraph that produces the weight for dynamic quant or weight only quant node and run the subgraph to observe the weight. Note that the observers of dynamic quant or weight only quant ops are run during the convert step. call_functionN) graphnodesoptargetr+ enumerateargsr)r*)r/r8iZnode_argZweight_observer_nodesZweight_observer_moduler5r5r6run_weight_observersQs rE) quantizedr0c Cs|}|jjD]}|jdkr |jtjkr t|j}t|dkr |D]@}|j ||j dtj|j |j }W5QRX| ||q@|j|q t||j|j}|S)Nr=r)r>r?r@rAtorchZquantize_per_tensor_dynamiclistusersleninserting_before create_noderCkwargsreplace_input_with erase_noder!preserved_attr_namesrFquantized_rootr8rIuserZnew_noder5r5r6duplicate_quantize_dynamic_nodehs"   rTc Cs|}|jjD]}|jdkr$|jdks:|jdkr |jtjkr t|j}t|dkr |D]<}|j ||j dd|j i}W5QRX| ||qT|j |q t||j|j}|S)z If a dequantize node has multiple uses, duplicate it and create one dequantize node for each use. This is to enable the pattern matching to map from individual quant - dequant - ref_module to final quantized module. call_method dequantizer=r)r>r?r@rArGrVrHrIrJrKrLrCrNrOr!rPrQr5r5r6duplicate_dequantize_node{s     rWc Cs|}|jjD]z}t|j}dd|jD}t|dkr |j||jdd|dji}W5QRX|D]}|||j |qjq t ||j|j }|S)z Removes duplicate dequant nodes in the graph, for an operator that has multiple dequant nodes as a user, replace them with a single dequant node that can be shared across all the uses. cSs:g|]2}|jdkr|jdks2|jdkr|jtjkr|qS)rUrVr=)r@rArGrV).0rSr5r5r6 s   z+remove_extra_dequantize..rrUrVr) r>r?rHrIrJinserting_afterrLrCreplace_all_uses_withrOr!rP)rFrRr8rIZ dequant_usersZ unique_dqZdequantr5r5r6remove_extra_dequantizes   " r\cCs|}|jjD]}|jdkr |jtjtjfkr t|j}|rB|dnd}t |dkr |jdkr |jdkr | |j d|j |t|j }|j ||D],}t |trt t|jdkr|j |qq t||j|j}|S)Nr=rrrUrV)r>r?r@rArGZquantize_per_tensorZquantize_per_channelrHrIrJr[rCrOr:r r!rP)rFrRr8rIrS orig_argsargr5r5r6remove_quant_dequant_pairss      r_)r^r8r>cCst|tr6|jdkr6|jdkr6|jd}|||n`t|ttfr\|D]}t|||qHn:t|t r| D]}t|||qnnt dt |dS)z If the arg is a dequantize Node, or a list/tuple/dict of dequantize Node, we'll recursively remove the dequantize Node rUrVrz6Unsupported node type in recursive remove dequantize: N)r:r r@rArCrNrHtuple!maybe_recursive_remove_dequantizedictvalueswarningswarntype)r^r8r> quantize_nodeZ arg_elementr5r5r6ras    ra)obs_noder3r9c Cs|jd}t|ts"td||j|kr:||jdknd}|rt|j}|rX|dnd}d}|D]&}|jdkrd|jt j j j krd|}qqd|r|}d} n|}d} |r|j|kr||j\} } nd} | | fS)aj Given and observer node, get the `Scope` or the fully qualified name for the submodule containing the observed node, also return a prefix of "_input" when the observed node is an input of a F.linear op, and not the output of another quantized op. TODO: this logic is hacky, we should think about how to remove it or make it more general rz.Expecting observed node to be a Node, but got Nr=_input) rCr:r r1r;rHrIr@rArGnnZ functionalZlinear) rhr3r9 observed_nodeZis_input_observer_onlyrIZfirst_linear_use_or_first_useZ linear_nodenprefix module_path_r5r5r6get_module_path_and_prefixs,   rqr8r>c CsL||8|d|f}t|jD]}||k r$|||q$W5QRXdS)z3 Inserts dequantize node for `node` in `graph` rVN)rZrUrbrIrN)r8r>Zdequantize_nodeZ user_noder5r5r6insert_dequantize_nodes  rs)r8modulesr0cCs@|jD]0\}}|jdkr |t|j}t|r |Sq dS)z\ If the node is observed, return the observer instance. Otherwise, return None. call_moduleN)rIitemsr@strrAr-)r8rtZmaybe_obs_noderp maybe_obsr5r5r6maybe_get_observer_for_nodes   ry)r8rtmodel is_referencebackend_config_dictcCstjjjj}|t|j}|j}t |j }t t |D]X} | |kr:|| } | j dkr:| jdkr:| j d} || | t | jdkr:|j| q:|j} t | dkr| ddkstdt||j||||d} t|j\}}t|||| | |t|j<dS)a Converts a observed standalone module to a quantized standalone module by calling the fx convert api, currently using the same `is_reference` flag as parent, but we may changing this behavior in the future (e.g. separating quantization and lowering for standalone module as well) Args: - node: The call_module node of the observed standalone module - modules: named_module of original model - model: original model - is_reference: a flag from parent provided by user to decide if we want to produce a reference model or a fbgemm/qnnpack model - backend_config_dict: backend configuration of the target backend of quantization rUrVrzCurrently only quantized)r{r|N)rGZaoZ quantizationZ quantize_fxZ convert_fxrwrAZ'_standalone_module_input_quantized_idxstolistrHrCrangerJr@rNrIr>rOZ(_standalone_module_output_quantized_idxsr1rsrsetattr)r8rtrzr{r|convertZobserved_standalone_moduleZsm_input_quantized_idxsrCidxr^rgZsm_output_quantized_idxsZquantized_standalone_module parent_namer;r5r5r6convert_standalone_modules4      r)r8rtr4r9r|cCs.|t|j}|j}d}t|}t||rV|j}|}t|j\} } t|| | ||j |k} |dksvt ||sv| szdSt |} | t |g} t|| sdSt|}|sdSd}|}t|tjjjr|}|d}i}t|tjjr(|}|}||j||jt|}t|}||d}nt|tjjr|jD]R}t||r>|dr>t||}|}|jtjkr||t|||<q>n$|dkr|}||jt|}t |}| t |d}|dk st!dt ||"||}|dk r ||d<nt|j\} } t|| | |dS)a Convert a weighted module to reference quantized module in the model If the QConfig of a QAT module is not set, the module will still be converted to a float module. Args: - node: The call_module node of the observed standalone module - modules: named_module of original model - observed_node_names: names for the set of observed fx node, we can skip this conversion if the node is not observed Nr) weight_ih weight_hhweightz3No reference quantized module class configured for )#rwrAqconfigrr:Zweight_fake_quantZto_floatrrr;r<rgetrfrrrGrkZ intrinsicZ _FusedModuleZ RNNCellBaserrrrZLSTMZ_flat_weights_nameshasattr startswithgetattrZdtypeZqint8rr1 from_float)r8rtr4r9r|Zoriginal_modulerZweight_post_processqat_module_classesrr;Z is_observedZpattern_to_dtype_configsZ dtype_configsZis_weight_quantizedZ fused_moduleZ float_moduleZ wq_or_wq_dictZweight_post_process_ihZweight_post_process_hhZweight_qparams_ihZweight_qparams_hhZwnr)root_module_to_quantized_reference_moduleZref_qmodule_clsZ ref_qmoduler5r5r6convert_weighted_moduleRst           r)r8r>rtcustom_module_class_mapping(statically_quantized_custom_module_nodescCs|t|j}t||}|j}t|r|||jd}t|tsRt d||j dkr|jdkr| ||jdt |j dkr||t||} | dk st | |_t|||} | |} t|j\} } t|| | | dS)a Converts an observed custom module to a quantized custom module based on `custom_module_class_mapping` For static quantization, we'll also remove the previous `dequantize` node and attach the observer node for output to the module, the observer for the node will be converted to a dequantize node instead of quantize-dequantize pairs later in the graph. In the end we would have a quantized custom module that has the same interface as a default quantized module in nn.quantized namespace, i.e. quantized input and quantized output. Args: - node: The call_module node of the observed standalone module - graph: The graph containing the node - modules: named_module of original model - custom_module_class_mapping: mapping from observed custom module class to quantized custom module class, used to swap custom modules - statically_quantized_custom_module_nodes: we'll add the custom module node if we find it is statically quantized, this will be used later when converting observers to quant/dequant node pairs, if the observed node is a statically quantized custom module nodes, we'll convert the observer to a dequantize node, this is to keep the interface the same as the default quantized module. TODO: maybe we want to redesign this part to align with reference model design as well, but there has been some discussions around the interface, so we can do it later. rzDExpecting the argument for custom module node to be a Node, but got rUrVN)rwrAryrraddrCr:r r1r@rNrJrIrOactivation_post_processrZ from_observedrr)r8r>rtrrZobserved_custom_modulerxrZ prev_noderZquantized_custom_module_classZquantized_custom_modulerr;r5r5r6convert_custom_modules2       rFT)rzr{convert_custom_config_dictis_standalone_module_remove_qconfig_flagconvert_qconfig_dictr|r0c& Cs`|dkr i}t|\}}} |j} |s,|t|jdd} |r|j} t| } t||j rht |i}t ||}t | |t || |j||}| D]N\}}||kstd|||dk rt|||std||||q|} t|d}|di}|jdk r"t|| }t|| |t|g}|jjD]}|jdkr6||jq6tjjt t!t"t#tjjft"t#t$t#t%fft"t#t&fddd d }t!t d d d }d}|dg}|dg}|dkrt'}t(|}t)|*}t+|}t,|}t-}t.|jjD]}|jdkrB|} |d7}| |krt/||jq |jdkrt0|dkr`q |}!|j1d}"t2|"t.t)fr|D]}#t3|"|#|!|jqn>t2|"t!tfrd|krt3|"|!|jnt45dt%|"q |jdkr t6| |j7r4|j1d}$|$|kr|||jn|||j|| || nt8| |j7rVt9|| |||nt%| |j7t-|:|:|krt%| |j7|krt%| |j7d|krq t;|| | | |n&t%| |j7|kr t<||j| ||q t-|dg}%t=|t|j|%}|j>|?|sNt@|}tA|}tB|| |}tC|}tD|}|r\tE||S)a We will convert an observed model (a module with observer calls) to a reference quantized model, the rule is simple: 1. for each observer module call in the graph, we'll convert it to calls to quantize and dequantize functions based on the observer instance 2. for weighted operations like linear/conv, we need to convert them to reference quantized module, this requires us to know whether the dtype configured for the weight is supported in the backend, this is done in prepare step and the result is stored in observed_node_names, we can decide whether we need to swap the module based on this set standalone_module means it a submodule that is not inlined in parent module, and will be quantized separately as one unit. Returns a quantized standalone module, whether input/output is quantized is specified by prepare_custom_config_dict, with input_quantized_idxs, output_quantized_idxs, please see docs for prepare_fx for details NF)Zremove_duplicatez&Expected key {} in convert qconfig_mapz~Expected k {} to have the same value in prepare qconfig_dict and convert qconfig_dict, found {} updated to {}.Z)observed_to_quantized_custom_module_class placeholder)rzr>r8rtr3r9r0c sZ|dk s tt|jtstt||\}}||j}t|} tfddt|jt|j D} | st| dkr| | | |jd| |W5QRXn| \} } } | ||jd}|g}| D]>\}}|dkrt||||||}||q||q|| | t|i}|jd|fd}| || |W5QRXdS)z Replace activation_post_process module call node with quantize and dequantize node Before: ... -> observer_0(x) -> ... After: ... -> torch.quantize_per_tensor(x, ...) -> x.dequantize() -> ... Ncsg|]}t|qSr5)r<)rXrmr9r5r6rYrszSconvert..replace_observer_with_quantize_dequantize_node..r)Z_scale_Z _zero_point_rV)rC)r1r:rArwrqr'allrHrCrIkeysrKr[rOrvr(appendrLr`rU)rzr>r8rtr3r9rornZobserver_moduleZmaybe_quantize_node_infoZskip_replacementZ node_typeZ quantize_opZqparamsZ input_nodeinputskeyvalueZ qparam_nodeZquantized_nodeZdequantized_noder5rr6.replace_observer_with_quantize_dequantize_node\s2            z?convert..replace_observer_with_quantize_dequantize_noderrcSsD|jd}t|ts"td|||||t||dS)Nrz@Expecting the for call custom module node to be a Node, but got )rCr:r r1r[rOrs)r8r>Zcall_custom_module_noder5r5r6%replace_observer_with_dequantize_nodes    z6convert..replace_observer_with_dequantize_noderinput_quantized_idxsoutput_quantized_idxsroutputz1Unsupported node type for output_quantized_idxs: rupreserved_attributes)Fr7Z _qconfig_mapcpurbZ named_modulesZ _qconfig_dictcopydeepcopyrZ_is_qatrrrrr>rvr1formatrr&rZ_equalization_qconfig_mapr$r%rEr?r@rr;rGrkModuler r rrwrrfrr rr`rrrsetrHrsrJrCr:rardrer-rAr#runionrrr!Zeliminate_dead_codeZ recompilerWrTr.r_r\r,)&rzr{rrrrr|r3r2r4r9rtZprepare_qconfig_dictZ modules_copyZconvert_qconfig_mapkvZcustom_module_classesrZweight_eq_obs_dictZ graph_inputsr8rrZplaceholder_node_seen_cntrrrZroot_module_classesrZfused_module_classesrZcur_placeholder_node_idxZ return_noderrrlrr5r5r6rs              9             r)FNFTNN)OtypingrrrrrrrrGrrdZtorch.fxr Ztorch.fx.graphr r r utilsrrrrrrrrZqconfig_dict_utilsrrZ qconfig_utilsrrrrZ*torch.ao.quantization.backend_config.utilsrrrrZ$torch.ao.quantization.backend_configr Z graph_moduler!r"r#Z _equalizer$r%r&r'r(r)r*r+Ztorch.ao.quantization.quantizer,r-r.rkrrwrfr7boolr<rErTrWr\r_rarqrsryZfxrrrrr5r5r5r6s$       0   ?  m  B