U d@ @sddlmZddlmZmZddlmZddlmZm Z m Z m Z m Z m Z mZmZddlZddlmZddlmmZddlmZddlZddlmZddlmmZddlZddl Z ddl!Z!ddl"m#Z#e$e e$e$fdd d Z%e eej&e e$e fd d d Z'ej&e e$e fejj(dddZ)d0ejj(ejj(dddZ*ej(ej(dddZ+ej(eej&eej&eej&dddZ,ej-ej.ej/ej0ej1ej2ej3ej4ej5ej6ej4ej7ej8g Z9ej:ej;gZej/ddiZ?eej&e e$ej(fdddZ@eej&e e$ej(fe ej(ej(fdd d!ZAGd"d#d#ZBd1d&d'ZCeBeDd(d)d*ZEGd+d,d,ZFdejGfejj(e e e$e feejGejj(d-d.d/ZHdS)2N)ArgumentTarget)fuse_conv_bn_eval)TypeDictAnyTupleIterableOptionalListcast) ShapeProp) defaultdict)Enum)targetreturncCs&|dd^}}|r|dnd|fS)zp Splits a qualname into parent path and last atom. For example, `foo.bar.baz` -> (`foo.bar`, `baz`) .r)rsplit)rparentnamerF/tmp/pip-unpacked-wheel-ua33x9lu/torch/fx/experimental/optimization.py _parent_namesr)patternnodemodulescCst|jdkrdS|jd|f}t||D]d\}}t|tjsDdS|jdkrTdSt|jtsfdS|j|krvdSt ||j|k r*dSq*dS)NrF call_moduleT) lenargszip isinstancefxNodeoprstrtype)rrrnodesZ expected_typeZ current_noderrrmatches_module_patterns    r))rr new_modulecCs<t|jtstt|j\}}|||j<t||||dSN)r"rr&AssertionErrorrsetattr)rrr* parent_namerrrrreplace_node_module,s r/F)modelrc Cstjtjftjtjftjtjfg}|s0t|}t |}t | }t|j }|D]}|jD]~}t|||r`t|jdjdkrq`||jdj}||j} | jsq`t|| } t|jd|| ||jd||q`qVt ||S)z Fuses convolution/BN layers for inference purposes. Will deepcopy your model by default, but can modify the model inplace as well. rr)nnZConv1dZ BatchNorm1dConv2d BatchNorm2dZConv3dZ BatchNorm3dcopydeepcopyr#symbolic_tracedict named_modulesgraphr(r)rr usersrZtrack_running_statsrr/replace_all_uses_with erase_node GraphModule) r0Zinplacepatternsfx_modelr new_graphrrconvZbnZ fused_convrrrfuse2s.           rBcCs*t|}Gdddtjj}||S)z5 Removes all dropout layers from the module. cs8eZdZeeedfeeefedfdd Z Z S)z&remove_dropout..DropoutRemover.)rr kwargsrcs>t|j|tjr*t|dks"t|dSt|||SdS)Nrr)r"Z submodulesr1ZDropoutrr,superr)selfrr rC __class__rrrVsz2remove_dropout..DropoutRemover.call_module) __name__ __module__ __qualname__rrrrr&rr __classcell__rrrFrDropoutRemoverUsrL)r#r6torchZ TransformerZ transform)r0r?rLrrrremove_dropoutOs rN) orig_moduler(inputsoutputscs|t}i|D]}||j}||<q|D] }||fdd}||<q.|fdd|D|t||S)z Given lists of nodes from an existing graph that represent a subgraph, returns a submodule that executes that subgraph. cs|Sr+r)xenvrrhz"extract_subgraph..csg|] }|qSrr).0outputrSrr jsz$extract_subgraph..)r#Graph placeholderrZ node_copyrXlintr=)rOr(rPrQr@inputZnew_noderrrSrextract_subgraph^s   r^cCs t|Sr+) th_mkldnnZMkldnnBatchNorm)a_rrrrU{rVrU)r(rcCsi}|D]r}|jdkrt|jts&t||j}t|tkrtt||tj}t|t j s`tt |||<t |||q|S)z For each node, if it's a module that can be preconverted into MKLDNN, then we do so and create a mapping to allow us to convert from the MKLDNN version of the module to the original. r)r%r"rr&r,r' mkldnn_maprMfloatr1Moduler4r5r/)r(r old_modulesr cur_moduler*rrrmodules_to_mkldnns   rg)r(rrecCsJ|D]@}|jdkrt|jts"t||j}||krt||||qdS)za Maps each module that's been changed with `modules_to_mkldnn` back to its original. rN)r%r"rr&r,r/)r(rrerrfrrr reset_moduless   rhc@seZdZejdddZdS) MklSubgraphfx_graphcCs||_g|_g|_g|_dSr+)rkr( start_nodes end_nodes)rErkrrr__init__szMklSubgraph.__init__N)rHrIrJr#rZrnrrrrrisri rcs(ddttdfdd }|S)aW This generates a heuristic that can be passed into `optimize_for_inference` that determines whether a subgraph should be run in MKL by running it with the example_inputs. Example usage: heuristic = gen_mkl_autotuner(example_inputs, iters=10) fast_model = optimization.optimize_for_inference(model, heuristic) Nr9rcs|j}dkr,|jj|jjtdd|Dtttj dd|j D}t |j ||fdd}|fdd}t jj t|fdd}||kS) NcSsg|]}t|jqSr)rMZrandnshaperWrrrrrYsz@gen_mkl_autotuner..use_mkl_heuristic..cSsg|]}|jdqS)r)r rrrrrrYscs<tD] }|qt}tD] }|}q$t|Sr+)rangetime)frabeginout)iterswarmuprr benchmarks   z?gen_mkl_autotuner..use_mkl_heuristic..benchmarkcsddddDDS)NcSsg|] }|qSrto_denserWirrrrYszRgen_mkl_autotuner..use_mkl_heuristic....cSsg|] }|qSr) to_mkldnnr}rrrrYsrrZ sample_inputs submodulerrrUrVz>gen_mkl_autotuner..use_mkl_heuristic..csSr+rrrrrrUrV)rlrkZ owning_modulerer propagater r r#r$rmr^r(rhr9r7r8)r9Z input_nodesZ output_argsrzZmkl_timeZ no_mkl_timeexample_inputsr?rxreryrruse_mkl_heuristicsz,gen_mkl_autotuner..use_mkl_heuristic)ribool)rrxryrrrrgen_mkl_autotuners rrpcCst|jdkS)z This is a heuristic that can be passed into `optimize_for_inference` that determines whether a subgraph should be run in MKL by checking if there are more than 2 nodes in it )rr()r9rrruse_mkl_lengthsrc@sBeZdZddZedddZeedddZeed d d Zd S) UnionFindcCsdg||_dg||_dS)Nrrsize)rEnrrrrns zUnionFind.__init__)vcCs||j|<d|j|<dS)Nrr)rErrrrmake_sets zUnionFind.make_set)rrcCsB|j|}||kr|S|dk s"t|||j|<tt|j|Sr+)rr,findr int)rErparrrrrs   zUnionFind.find)r`bcCsf||||}}||kr"|S|j||j|kr@||}}||j|<|j||j|7<dSr+)rrr)rEr`rrrrjoins  zUnionFind.joinN)rHrIrJrnrrrrrrrrrsr)r0 pass_configtracerrc s*dddtid}|dkri}|||dr6t|}|drFt|}|ddkrV|St|dtsltd d|dkrtd |dd}|}|t |t |j }t| }Gd d d t}tjD]V} |j} | jd krV|| j} t| tkr|j} t| d} | dk r| jtjks:td| jtdkstdn2| jdkr| jtkrv|j} n| jtkr|j} | |jkr| |jkrt dd| j!Dsqֈ"| t #| j!fdd} W5QRXt$t%t j&j'| | _!(| ()dd| f}| *|| f|_!W5QRXqt+tj|}|_,jD]} | jdkrL| jdkrL| j!d}t| j-}|D]2}|jdkr|jdkr|*|.|qt/| j-dkrL.| qLt/j}t0|fddt1jD]\}} | jdkr,| jdkr,|| _23|n| jdkrn| jdkrn| j!ddk s\t| j!d| _4ntfdd| j5D}t/|dkrqt dd|Drtt6|}|d| _7|ddD]}8|d|qʐqt9fd d}jD]r} t:| d!r$|;| j7j<| t:| d"rH|;| j2j=<| t:| d#r|;| j4j><| q|?D]P}||sx|j=|j>D]$} | j!d}| *|.| qt@|j||qxd}jD]&} | jdks| jdkr|d7}qtABtCDd$|Et |}|S)%a Performs a set of optimization passes to optimize a model for the purposes of inference. Specifically, the passes that are run are: 1. Conv/BN fusion 2. Dropout removal 3. MKL layout optimizations The third optimization takes a function `use_mkl_heuristic` that's used to determine whether a subgraph should be explicity run in MKL layout. Note: As FX does not currently handle aliasing, this pass currently assumes nothing aliases. If that isn't true, use at your own risk. T heuristic) conv_bn_fuserNmkldnn_layout_optimizeNrrNrFz+mkldnn_layout_optimize config is not a dictz4Heuristic not found in mkldnn_layout_optimize configc@seZdZdZdZdZdS)z*optimize_for_inference..MklSupportrrN)rHrIrJNOYESUNKNOWNrrrr MklSupportsrrz)this pass is only for torch.float modulescpuz!this pass is only for CPU modules call_functioncSsg|]}|jdkqSr{)r)rWargrrrrY3sz*optimize_for_inference..csd|fS)Nr) call_methodrrjrrrU6rVz(optimize_for_inference..rr|rrcs0t|dr|jSt|dr,|jSdS)Ncolor start_color)hasattrrrrr)ufrr get_colorSs     z)optimize_for_inference..get_colorcs,g|]$}t|tjr|dk r|qSr+)r"r#r$r})rrrrYns css|]}|dkVqdSr+rr}rrr rsz)optimize_for_inference..rcstSr+)rirrjrrrUyrVrr end_colorzmkldnn conversions: )FrupdaterBrNr"r7 RuntimeErrortracer4r5r#r=rootr8rlistr(rr%rr'mkldnn_supportedrnext parametersZdtyperMrcr,Zdevicemkldnn_supported_unknownranyr Zinserting_beforeZmap_argr rrrZinserting_afterZ create_noder;rgrer:r<rr enumeraterrrZall_input_nodessortedrrrrrappendrlrmvaluesrhlogging getLoggerrHinfor\)r0rrZdefault_pass_configrZ cur_tracerr?rrrZsupports_mkldnnrfZsample_parameterZ mkldnn_argsZdense_xreZprv_noder:userZ num_nodesZcur_idxZ cur_colorsZ other_colorZ mkldnn_graphsr9ZprvZmkldnn_conversionsresultr)rkrrroptimize_for_inferences                                     r)F)ror)IZtorch.fxr#Z torch.fx.noderrZtorch.nn.utils.fusionrtypingrrrrr r r r rMZtorch.nnr1Ztorch.nn.functionalZ functionalFZtorch.fx.passes.shape_propr r4 collectionsrZtorch.utils.mkldnnutilsZmkldnnr_operatorrtrenumrr&rr$r)rdr/rBrNr^r2ZLinearr3ZReLUZ MaxPool2dZ AvgPool2dZAdaptiveAvgPool2dZreluZ transposeZsigmoidZ avg_pool2dZadaptive_avg_pool2draddmulrZ MkldnnConv2dZ MkldnnLinearrbrgrhrirrrrZTracerrrrrrsp  (      (  .  &