U 9%e @sRddlZddlZddlmZmZmZddlZdddddgZdZ dd Z d d Z d d Z ddZ eejeejfddddZde ddfddZdde ddfddeejeeejeejfeeeeejjeeejddddZdeejeddddZdeejeejddddZe ddfeejeejeddddZdS)N)OptionalSequenceUnion all_reducereduce broadcast all_gatherreduce_scattercCstttjdstddSt}|D]J}|jr4dS|sBdS|jsNdS| }||krddS| |q$dS)N_nccl_all_reducez)PyTorch is not compiled with NCCL supportFT) hasattrtorch_CwarningswarnsetZ is_sparseZ is_contiguousZis_cudaZ get_deviceadd)ZtensorsZdevicesZtensorZdevicerN/var/www/html/Darija-Ai-API/env/lib/python3.8/site-packages/torch/cuda/nccl.py is_available s    rcCs0tj}|d?}|d?d@}|d@}|||fS)N i)r r Z _nccl_version)vermajorminorpatchrrrversion"s   rcCs tjSN)r r Z_nccl_unique_idrrrr unique_id*srcCstj|||Sr)r r Z_nccl_init_rank)Z num_ranksuidZrankrrr init_rank.sr)inputsreturncCs&t|tjjrt|tjr"tddS)Nz(Inputs should be a collection of tensors) isinstance collectionsabc Containerr Tensor TypeError)r rrr_check_sequence_type2s r(cCs4t||dkr|}t|tj|||||dSr)r(r r r r outputsopstreamscommsrrrr9s )r*)r outputrootr+r,r*r!cCst||dk r6|dk r"tdqxtd||}nBt|tjsdt|tjj rdtd||}n|dkrt||n|}tj ||||||dS)Nz'output' and 'outputs' can not be both specified. 'outputs' is deprecated in favor of 'output', taking in a single output tensor. The signature of reduce is: reduce(inputs, output=None, root=0, op=SUM, streams=None, comms=None).znccl.reduce with an output tensor list is deprecated. Please specify a single output tensor with argument 'output' instead instead.z\nccl.reduce with an output tensor list is deprecated. Please specify a single output tensor.) r( ValueErrorrrr"r r&r#r$rr Z _nccl_reduce)r r.r/r+r,r-r*_outputrrrrCs(   )r r/r!cCst|tj||||dSr)r(r r Z_nccl_broadcast)r r/r,r-rrrrjs)r r*r!cCs&t|t|tj||||dSr)r(r r Z_nccl_all_gather)r r*r,r-rrrrqs)r r*r+r!cCs(t|t|tj|||||dSr)r(r r Z_nccl_reduce_scatterr)rrrr |s)rNN)NN)r#rtypingrrrZ torch.cudar __all__ZSUMrrrrr&r(rintcudaZStreamrrrr rrrrsf    (