o vžÄi>ã@sFddlmZddlmZddlmZddlmZeGdd„dƒƒZdS)é)Ú dataclass)Úcached_property)Úinit_device_mesh)Úlogc@s¾eZdZUeed<eed<eed<eed<eed<eed<eed<dd „Zd d „Zd d „Ze dd„ƒZ e dd„ƒZ e dd„ƒZ e dd„ƒZ e dd„ƒZe dd„ƒZe dd„ƒZedd„ƒZdS)Ú ParallelDimsÚ dp_replicateÚdp_shardÚcpÚtpÚppÚ world_sizeÚenable_loss_parallelcCs | ¡dS©N)Ú _validate©Úself©rú[/data/cameron/vidgen/cosmos-policy/cosmos_policy/_src/reason1/parallelisms/parallel_dims.pyÚ __post_init__"s zParallelDims.__post_init__cCs0|j|j|j|j|jf\}}}}}||||fD] }|dks!Jdƒ‚q|dks.|dks.Jdƒ‚|dkr\t d|j›d||||›d¡|j|||||_}t d |›d¡|dksbJ‚||||||jks–|j|||||_t d |›d |›d |›d |›d|›d|j›d ¡dSdS)Néz6Parallelism degree should be >= 1, except for dp_shardéÿÿÿÿz dp_shard must -1 or >=1.rzLdp_shard is set to -1, will be automatically determined based on world_size z // Ú.zdp_shard is set to z$Invalid parallel dims: dp_replicate(z ) * dp_shard(z) * cp(z) * tp(z) * pp(z) != WORLD_SIZE(ú)) rrr r r rÚinfor Úwarning)rrrr r r Údrrrr%s<ûÿ ÿÿÿÿÿþzParallelDims._validatec CsRg}g}t|j|j|j|j|jggd¢ƒD]\}}|dkr'| |¡| |¡qt dt |ƒ›d|›d|›¡t |ƒ}t |||d}g}g}g} |j rV| d¡|  d¡|j rh| d¡| d¡|  d¡|jru| d ¡|  d ¡|gkrƒ|t |ƒjd d |gkr‘|t |ƒjd d | gkrŸ|t | ƒjd d t d|›¡|S)N)r rrr r rz Building z-D device mesh with z, )Úmesh_dim_namesrrr Údp)Ú mesh_dim_nameÚ dp_shard_cpÚdp_cpzmesh: )Úzipr rrr r ÚappendrrÚlenÚtuplerÚdp_replicate_enabledÚdp_shard_enabledÚ cp_enabledÚ_flatten) rÚ device_typeÚdimsÚnamesrÚnameÚmeshZdp_mesh_dim_namesZdp_shard_cp_mesh_dim_namesZdp_cp_mesh_dim_namesrrrÚ build_mesh@sD þ  €        zParallelDims.build_meshcCs|jdkp |jdkS©Nr)rrrrrrÚ dp_enabledlszParallelDims.dp_enabledcCó |jdkSr/)rrrrrr%pó z!ParallelDims.dp_replicate_enabledcCr1r/)rrrrrr&tr2zParallelDims.dp_shard_enabledcCr1r/)r rrrrr'xr2zParallelDims.cp_enabledcCr1r/)r rrrrÚ tp_enabled|r2zParallelDims.tp_enabledcCr1r/)r rrrrÚ pp_enabled€r2zParallelDims.pp_enabledcCs|jdko|jSr/)r r rrrrÚloss_parallel_enabled„sz"ParallelDims.loss_parallel_enabledcCs|j|j|jSr)r r r rrrrÚnon_data_parallel_sizeˆsz#ParallelDims.non_data_parallel_sizeN)Ú__name__Ú __module__Ú __qualname__ÚintÚ__annotations__Úboolrrr.Úpropertyr0r%r&r'r3r4r5rr6rrrrrs6 ,       rN) Ú dataclassesrÚ functoolsrÚtorch.distributed.device_meshrÚ#cosmos_policy._src.imaginaire.utilsrrrrrrÚs