U
    d3                  	   @   s   d dl Z d dlmZ d dlmZmZmZmZmZ d dl	m
Z
 d dlZe
ddG dd dZe
dddee jjee jjjgef eeeef  d	d
dZdS )    N)GraphModule)CallableListDictAnyOptional)compatibilityT)Zis_backward_compatiblec                   @   s(   e Zd ZedddZedddZdS )	Partitionnamec                 C   sN   || _ d| | _g | _i | _i | _i | _i | _tjj	
 | _	i | _i | _d S )NZsubmod_)r   submod_name
node_namesinputsoutputspartitions_dependent_onpartition_dependentstorchfxgraphGraphenvironmenttargets)selfr    r   @/tmp/pip-unpacked-wheel-ua33x9lu/torch/fx/passes/split_module.py__init__	   s    zPartition.__init__)returnc                 C   s4   d| j  d| j d| j d| j d| j d| j S )Nzname: z
,
 nodes: z,
 inputs: z,
 outputs: z,
 partitions depenent on: z,
 parition dependents: )r   r   r   r   r   r   )r   r   r   r   __repr__   s    zPartition.__repr__N)__name__
__module____qualname__strr   r   r   r   r   r   r	      s   r	   )mroot_msplit_callbackqualname_mapc              	      sN  i i t jjjtt jjj dfdd| jjD ]j< jdkrNq4jdkrxt jj	j
d fdd q4t|}|d	krt| |< jj |_t jj	j
fd
d t jj	jfdd q4g } D ] \}tjs || q g }|rz| }|| | jD ],}| j| | jsH|| qHq&t|tkrtd|D ]F}| jD ]0}	j|	}
|	 j |
_|
j|	 < qq| jjD ]2tdrj jt jj	j
fdd}t jj	jfdd}jdkrRj}n|jd}| }|D ].}t||stdj dt ||}qfd!|}|j"|< |d	k	rΈj# d| }j||< t$|t%st&t$|t'st&jj(j|||d}j |_|j< qi  t jj) }i }| jjD ]܉jdkrtj
dkrdj
d nt*j+j,}|jjj-|d j< j  j _nzjdkr:|.j j< j  j _| }jdD ].}t||stdj dt ||}q||j< q:|D ]}| t%fddj/D }t|dkrX|d n|}j0| t jj12j"j|j#< |3j#t% fddjD }tj/dkrt jj45|}t6j/D ]\}}|| j |< qn| t7j/d < q| jjD ]6jdkr|0t jj	j
d  fdd qt jj12||S )aU  
    Creates subgraphs out of main graph

    Args:
        m (GraphModule): Graph module to split
        root_m (torch.nn.Module): root nn module. Not currently used. Included
            because the root nn module is usually transformed via
            torch.fx._symbolic_trace.symbolic_trace (see example below)
        split_callback (Callable[[torch.fx.node.Node], int]): Callable function
            that maps a given Node instance to a numeric partition identifier.
            split_module will use this function as the policy for which operations
            appear in which partitions in the output Module.
        qualname_map: Optional[Dict[str, str]]: optional output parameter that returns a
            mapping from new target names in the module after split to old target
            names in the original module.

    Returns:
        GraphModule: the module after split.

    Example:

        This is a sample setup:

            import torch
            from torch.fx.symbolic_trace import symbolic_trace
            from torch.fx.graph_module import GraphModule
            from torch.fx.node import Node
            from torch.fx.passes.split_module import split_module

            class MyModule(torch.nn.Module):
                def __init__(self):
                    super().__init__()
                    self.param = torch.nn.Parameter(torch.rand(3, 4))
                    self.linear = torch.nn.Linear(4, 5)

                def forward(self, x, y):
                    z = self.linear(x + self.param).clamp(min=0.0, max=1.0)
                    w = self.linear(y).clamp(min=0.0, max=1.0)
                    return z + w

            # symbolically trace model
            my_module = MyModule()
            my_module_traced = symbolic_trace(my_module)

            # random mod partitioning
            partition_counter = 0
            NPARTITIONS = 3

            def mod_partition(node: Node):
                global partition_counter
                partition = partition_counter % NPARTITIONS
                partition_counter = (partition_counter + 1) % NPARTITIONS
                return partition

            # split module in module with submodules
            module_with_submodules = split_module(
                my_module_traced, my_module, mod_partition
            )

        Output looks like this. Original graph is broken into partitions

            > print(module_with_submodules)
            GraphModule(
                (submod_0): GraphModule(
                    (linear): Linear(in_features=4, out_features=5, bias=True)
                )
                (submod_1): GraphModule(
                    (linear): Linear(in_features=4, out_features=5, bias=True)
                )
                (submod_2): GraphModule()
            )

            def forward(self, x, y):
                param = self.param
                submod_0 = self.submod_0(x, param, y);  x = param = y = None
                getitem = submod_0[0]
                getitem_1 = submod_0[1];  submod_0 = None
                submod_1 = self.submod_1(getitem, getitem_1);  getitem = getitem_1 = None
                getitem_2 = submod_1[0]
                getitem_3 = submod_1[1];  submod_1 = None
                submod_2 = self.submod_2(getitem_2, getitem_3);  getitem_2 = getitem_3 = None
                return submod_2

        Output of split module is the same as output of input traced module.
        This is an example within a test setting:

            > orig_out = my_module_traced(x, y)
            > submodules_out = module_with_submodules(x, y)
            > self.assertEqual(orig_out, submodules_out)
            True
    )def_nodeuse_nodec                    s   t | dd }t |dd }||kr|d k	rR | }|j| j |d k	rR|j| |d k	r | }|j| j |d k	r|j| d S )N_fx_partition)getattrr   
setdefaultr   r   r   r   )r&   r'   Zdef_partition_nameZuse_partition_nameZdef_partitionZuse_partition)
partitionsr   r   record_cross_partition_use   s    z0split_module.<locals>.record_cross_partition_use)placeholderget_attroutputr   c                    s
    | d S Nr   n)r,   r   r   <lambda>       zsplit_module.<locals>.<lambda>Nc                    s
   |  S r0   r   r&   noder,   r   r   r3      r4   c                    s
   |  S r0   r   r5   r6   r   r   r3      r4   z cycle exists between partitions!r(   c                    s    |  S r0   r   r1   r   r   r   r3      r4   c                    s    |  S r0   r   r1   r8   r   r   r3      r4   )call_moduler.   .zOperator target z not found!_)optargetargskwargsr-   )Z	type_exprdefault_valuer.   zNode target c                 3   s   | ]}j  |  V  qd S r0   r8   .0r   )
orig_nodes	partitionr   r   	<genexpr>  s     zsplit_module.<locals>.<genexpr>   c                 3   s   | ]} | V  qd S r0   r   rA   base_mod_envr   r   rE     s     c                    s
    | j  S r0   r
   r1   rG   r   r   r3     r4   )8r   r   r7   Noder   r   Znodesr   r<   Zmap_argr>   r!   getr	   r   appendr(   r?   itemslenr   popr   RuntimeErrorr   r-   metacopyr   hasattrr=   splitr)   joinr   r   
isinstancetupleAssertionErrordictZcreate_noder   inspect	Signatureemptytyper.   r   r/   Zgraph_moduler   r9   proxyZProxy	enumeratelist)r"   r#   r$   r%   Zpartition_nameZroot_partitionsZsorted_partitionsZroot_partitionZ	dependentinputr-   Zgathered_argsZgathered_kwargsr=   Ztarget_atomsZtarget_attrZatomqualnameZnew_nodeZbase_mod_graphZbase_mod_attrsr@   Zattr_valZoutput_valsZ
output_valZoutput_val_proxyiZoutput_namer   )rH   r   r7   rC   rD   r+   r,   r   split_module   s    b$










"  "(rc   )N)r   Ztorch.fx.graph_moduler   typingr   r   r   r   r   Ztorch.fx._compatibilityr   rY   r	   nnModuler   r7   rI   intr!   rc   r   r   r   r   <module>   s    