U
    dM                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlmZmZmZ d dl	m
Z
mZmZmZmZ d dlmZmZmZ d dlmZmZmZmZmZmZ d dlmZ d dlmZmZmZm Z m!Z! d dl"m#Z# d d	l$m%Z% d dl&m'Z' d dl(m)  m*Z* d d
l+m,Z, eddddddgZ-e.eeeef  eeef e-dddZ/d(ee eeef eee e.e.ee
e  ddddZ0ddddZ1d)e!e.e.eeef eeeef  eee e.dd	ddZ2d*e!e.e.e.eeef eeeef  eed e3e3e.e3dd!d"d#Z4d+e.e.e3ee. dd$d%d&Z5e6d'kre1  dS ),    N)
namedtupleCounterdefaultdict)ListDictUnionSequenceOptional)get_grouped_native_functionsparse_native_yamlNamespaceHelper)BackendIndexBackendMetadataDispatchKeyNativeFunctionNativeFunctionsGroupOperatorName)SelectiveBuilder)Target	concatMapcontext
YamlLoaderFileManager)native_function_manager)CodeTemplate)DispatcherSignatureParsedExternalYamlbackend_keyautograd_key
class_namecpp_namespacebackend_indices)backend_yaml_pathgrouped_native_functionsr!   returnc              	      s  dd t dd |D t| d}tj|td}W 5 Q R X t|tsJtddd	d
dddg}|dd d k	sxtd|dd }|d	d }|d k	std|dd}t|t	std| |dd}	t|	t	std|	 |dg }
|
d krg }
t|
t
s(td|
 dt|
 d|dg }t|t
sNtd| |dg }|
| t| dkst|  dd|  dd| tt tt	t	tdfdd}d t|
dkrtfd d tW 5 Q R X ||
||	d!}kst|< d  t|dkr~tfd"d td#  W 5 Q R X || ||	d!} ksvt| < |D ]}t|trd krg nd$d%  |fD } d krg nd&d%   |fD }n`d krg n d'd% fd(d%| D D } d kr$g n d)d%  fd*d%| D D }d+d% |D }d,d% |D }t|dkst|dkstd-|d j d.|d j d/qt ||S )0Nc                 S   s   i | ]}|j j|qS  funcname.0fr%   r%   >/tmp/pip-unpacked-wheel-ua33x9lu/torchgen/gen_backend_stubs.py
<dictcomp>,   s    z&parse_backend_yaml.<locals>.<dictcomp>c                 S   s   t | tr| gS t|  S N)
isinstancer   list	functionsr+   r%   r%   r,   <lambda>/       z$parse_backend_yaml.<locals>.<lambda>r)Loaderbackendr   r    extra_headers	supportedZautogradfull_codegenz&You must provide a value for "backend"z,You must provide a value for "cpp_namespace"use_out_as_primaryFzHYou must provide either True or False for use_out_as_primary. Provided: device_guardzBYou must provide either True or False for device_guard. Provided: z,expected "supported" to be a list, but got: z
 (of type )z+expected "autograd" to be a list, but got: r   z contains unexpected keys: z, z). Only the following keys are supported: )backend_opsdispatch_keyr;   use_device_guardr$   c          	         sd   i }| D ]H}t |}| ks,td| t | j}t|dd}|||< qt||d||dS )Nz Found an invalid operator name: F)kernelZ
structuredT)r?   r;   Zexternalr<   index)r   parseAssertionError
dispatcherr(   r'   r   r   )	r>   r?   r;   r@   metadataopZop_nameZkernel_namem)native_functions_mapr%   r,   create_backend_indexj   s"    

z0parse_backend_yaml.<locals>.create_backend_indexc                      s   d  dS )NzFThe provided value for "backend" must be a valid DispatchKey, but got .r%   r%   r7   r%   r,   r3      r4   )r;   r@   c                      s   d  dS )NzThe "autograd" key was specified, which indicates that you would like to override the behavior of autograd for some operators on your backend. However "Autogradz" is not a valid DispatchKey.r%   r%   rL   r%   r,   r3      s   ZAutogradc                 S   s   g | ]}|d k	r|qS r.   r%   r*   rH   r%   r%   r,   
<listcomp>   s   z&parse_backend_yaml.<locals>.<listcomp>c                 S   s   g | ]}|d k	r|qS r.   r%   rM   r%   r%   r,   rN      s   c                 S   s   g | ]}|d k	r|qS r.   r%   rM   r%   r%   r,   rN      s   c                    s   g | ]}   |qS r%   
get_kernelr)   )r!   r   r%   r,   rN      s   c                 S   s   g | ]}|d k	r|qS r.   r%   rM   r%   r%   r,   rN      s   c                    s   g | ]}   |qS r%   rO   r)   )r   r!   r%   r,   rN      s   c                 S   s   g | ]}|d k	r|qS r.   r%   r)   r%   r%   r,   rN      s      c                 S   s   g | ]}|d k	r|qS r.   r%   r)   r%   r%   r,   rN      s      zCurrently, all variants of an op must either be registered to a backend key, or to a backend's autograd key. They cannot be mix and matched. If this is something you need, feel free to create an issue! z" is listed under "supported", but z is listed under "autograd".)r   openyamlloadr   r/   dictrD   popboolr0   typeextendlenkeysjoinr   strr   r   r   rC   r   rP   r1   rA   r   )r"   r#   r!   r+   Zyaml_valuesZ
valid_keysr   r    r;   r@   r9   Zsupported_autogradr:   rJ   Zbackend_idxZautograd_idxgZforward_kernelsZbackward_kernelsr%   )r   r7   r!   r   rI   r,   parse_backend_yaml&   s   
  
  





    r^   )native_functionsr!   r   r   r   kernel_defn_file_pathr:   r$   c                    sl  z"t |d}| }W 5 Q R X W n" tk
rD   td| Y nX d krRg |d krpt|| j g  nt|| j fdd| D }	tt}
|	D ]}|
t	|j
 | q| d}tt||}d}|
 D ]p\}}t|}|| }||krttddd	 d
 fdd|D }|d
| d| d| d| d| d7 }q|dksht|d S )Nr5   z2Unable to read from the specified impl_path file: c                    s(   g | ] }|j j kr|j jkr|qS r%   r&   r)   )expected_backend_op_namesr:   r%   r,   rN      s    z,error_on_missing_kernels.<locals>.<listcomp>z::([\w\d]*)\([^\)]*\)\s*{ )r+   r$   c              
   S   s0   t |  t| j W  5 Q R  S Q R X d S r.   )r   r   Zfrom_schemar'   declr2   r%   r%   r,   create_decl  s    
z-error_on_missing_kernels.<locals>.create_decl
c                    s   g | ]} |qS r%   r%   r)   )rd   r%   r,   rN     s     z$ is missing a kernel definition for z. We found z( kernel(s) with that name,
but expected zH kernel(s). The expected function schemas for the missing operator are:
z

)rQ   readIOErrorrD   r0   rB   rZ   r   rE   r(   r'   appendr   refindallitemsrY   r   r\   r[   )r_   r!   r   r   r   r`   r:   r+   Zbackend_defnsZexpected_backend_native_funcsZ#expected_backend_kernel_name_countsZnative_fZkernel_defn_regexZ!actual_backend_kernel_name_countsZmissing_kernels_err_msgZexpected_namefuncsZexpected_overload_countZactual_overload_countZexpected_schemas_strr%   )rd   ra   r:   r,   error_on_missing_kernels   s\    	


rm   )r$   c                  C   sr   t jdd} | jdddd | jddd	d | jd
tdd	d | jdtd dd |  }t|j|j|j	|j
 d S )NzGenerate backend stub files)descriptionz-sz--source_yamlzApath to source yaml file containing operator external definitions)helpz-oz--output_dirzoutput directoryz	--dry_runF)rW   defaultro   z--impl_pathz9path to the source C++ file containing kernel definitions)argparseArgumentParseradd_argumentrV   r\   
parse_argsrunsource_yaml
output_dirdry_run	impl_path)parseroptionsr%   r%   r,   main  s     r|   rb   )	fmr   r    r!   r#   backend_dispatch_keyautograd_dispatch_keybackend_namer$   c                    s   d k	st dttttfdd|ttttfdd| t||  dd fdd d S )NzAAutogenerated file by gen_backend_stubs.py. Do not edit directly!c                    s   t |   S r.   destZ#compute_native_function_declarationr2   )r~   r!   r%   r,   r3   I  s    z4gen_dispatchkey_nativefunc_headers.<locals>.<lambda>c                    s    d krg S t |   S r.   r   r2   )r   r!   r%   r,   r3   U  s     NativeFunctions.hzDispatchKeyNativeFunctions.hc                      s   j j  dS )N)generated_commentZnamespace_prologuer   Znamespace_epilogueZdispatch_declarationsZBackendNamer   )Zprologueepiloguer%   )autograd_declarationsbackend_declarationsr~   r   r   r   	ns_helperr%   r,   r3   d  s    )rD   r0   sortedsetr   r   write_with_template)r}   r   r    r!   r#   r~   r   r   r%   )	r   r   r   r~   r!   r   r   r   r   r,   "gen_dispatchkey_nativefunc_headers3  s:    
r   FTr   )r}   rw   r   r    r!   r#   r~   r?   selectorbuild_in_treeper_operator_headersr   eager_registrationr$   c                    s   | d| dg}|	r,d dd |D nd dd |D d k	sLt|  tttj tjd dd}d	d		|rtd
}|j	|d	ntd}|j	||d| 
d dd 	f
dd d S )N/r   re   c                 s   s   | ]}d | dV  qdS )z
#include <>Nr%   r*   hr%   r%   r,   	<genexpr>  s     z/gen_dispatcher_registrations.<locals>.<genexpr>c                 s   s   | ]}d | dV  qdS )z
#include ""Nr%   r   r%   r%   r,   r     s     Frocmr    Zclass_method_nameZskip_dispatcher_op_registrationrb   zPTORCH_LIBRARY_IMPL(aten, $dispatch_key, m) {
    $dispatch_registrations_body
};)r?   dispatch_registrations_bodyzTORCH_API void Register${backend_name}${dispatch_key}NativeFunctions() {
    static auto m = MAKE_TORCH_LIBRARY_IMPL(aten, $dispatch_key);
    $dispatch_registrations_body
})r   r?   r   ZRegisterz.cppzRegisterDispatchKey.cppc                      s\   	dsdnd  tj ddt dtttj tjd dddS )Nrb   z#include <ATen/Functions.h>F)r   r   r   )"static_init_dispatch_registrationsdeferred_dispatch_registrationsZextra_cuda_headersZexternal_backend_headersZops_headersr   Zdispatch_namespaceZdispatch_headersZdispatch_helpersZdispatch_namespaced_definitionsZdispatch_anonymous_definitions)	lowerr   Zgen_registration_headersZgen_registration_helpersr0   r   RegisterDispatchKeyr   ZANONYMOUS_DEFINITIONr%   
Zbackend_indexr   r    r   r?   Zexternal_backend_headers_strr#   r   r   r   r%   r,   r3     s>      	z.gen_dispatcher_registrations.<locals>.<lambda>)r[   rD   r0   r   r   r   r   ZREGISTRATIONr   
substituter   )r}   rw   r   r    r!   r#   r~   r?   r   r   r   r   r   headersr   Zstatic_templateZdeferred_templater%   r   r,   gen_dispatcher_registrationsp  sX    	
r   )rv   rw   rx   ry   r$   c                    sB  t tjj }tj|dtt	d fdd}||}tj|d}tj|d}t
||}	|	j|	j }
}t|
}t| ||}|j}|j}|j}|j}|j}t }|d krd S |d kr||  }|d k	st|d k	rt|
||||| t||||||| |d kr|gn||gD ]}t|||||||||	 qd S )Nzaten/src/ATen/templates)install_dirr$   c                    s   t |  dS )N)r   template_dirrx   )r   )r   rx   r   r%   r,   make_file_manager  s
      zrun.<locals>.make_file_managerz*aten/src/ATen/native/native_functions.yamlzaten/src/ATen/native/tags.yaml)pathlibPath__file__parentabsoluteospathr[   r\   r   r   r_   r!   r
   r^   r   r   r    r   r   Zget_nop_selectorZnative_function_class_namerD   rm   r   r   )rv   rw   rx   ry   Zpytorch_rootr   r}   Znative_yaml_pathZtags_yaml_pathZparsed_yamlr_   r!   r#   Zparsed_backend_yamlr   r   r    r   r   r?   r%   r   r,   ru     sz     
  	ru   __main__)N)rb   )FFrb   T)N)7r   rq   r   rR   ri   collectionsr   r   r   typingr   r   r   r   r	   Ztorchgen.genr
   r   r   Ztorchgen.modelr   r   r   r   r   r   Z!torchgen.selective_build.selectorr   Ztorchgen.utilsr   r   r   r   r   Ztorchgen.contextr   Ztorchgen.code_templater   Ztorchgen.destr   Ztorchgen.api.dispatcherapirE   Ztorchgen.api.typesr   r   r\   r^   rm   r|   r   rV   r   ru   __name__r%   r%   r%   r,   <module>   s    
 B 

? 
H    
i    T
