o ?߱i@@sFddlmZddlmZddlmZddlmZeGdddZdS)) dataclass)cached_property)init_device_mesh)logc@seZdZUeed<eed<eed<eed<eed<eed<eed<dd Zd d Zd d Ze ddZ e ddZ e ddZ e ddZ e ddZe ddZe ddZeddZdS) ParallelDims dp_replicatedp_shardcptppp world_sizeenable_loss_parallelcCs |dSN) _validateselfra/data/cameron/vidgen/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py __post_init__"s zParallelDims.__post_init__cCs0|j|j|j|j|jf\}}}}}||||fD] }|dks!Jdq|dks.|dks.Jd|dkr\td|jd||||d|j|||||_}td |d|dksbJ||||||jks|j|||||_td |d |d |d |d|d|jd dSdS)Nz6Parallelism degree should be >= 1, except for dp_shardz dp_shard must -1 or >=1.rzLdp_shard is set to -1, will be automatically determined based on world_size z // .zdp_shard is set to z$Invalid parallel dims: dp_replicate(z ) * dp_shard(z) * cp(z) * tp(z) * pp(z) != WORLD_SIZE()) rrr r r rinfor warning)rrrr r r drrrr%s< zParallelDims._validatec CsRg}g}t|j|j|j|j|jggdD]\}}|dkr'||||qtdt |d|d|t |}t |||d}g}g}g} |j rV|d| d|j rh|d|d| d|jru|d | d |gkr|t |jd d |gkr|t |jd d | gkr|t | jd d td||S)N)r rrr r rz Building z-D device mesh with z, )mesh_dim_namesrrr dp) mesh_dim_name dp_shard_cpdp_cpzmesh: )zipr rrr r appendrrlentuplerdp_replicate_enableddp_shard_enabled cp_enabled_flatten) r device_typedimsnamesrnamemeshZdp_mesh_dim_namesZdp_shard_cp_mesh_dim_namesZdp_cp_mesh_dim_namesrrr build_mesh@sD           zParallelDims.build_meshcCs|jdkp |jdkSNr)rrrrrr dp_enabledlszParallelDims.dp_enabledcC |jdkSr/)rrrrrr%p z!ParallelDims.dp_replicate_enabledcCr1r/)rrrrrr&tr2zParallelDims.dp_shard_enabledcCr1r/)r rrrrr'xr2zParallelDims.cp_enabledcCr1r/)r rrrr tp_enabled|r2zParallelDims.tp_enabledcCr1r/)r rrrr pp_enabledr2zParallelDims.pp_enabledcCs|jdko|jSr/)r r rrrrloss_parallel_enabledsz"ParallelDims.loss_parallel_enabledcCs|j|j|jSr)r r r rrrrnon_data_parallel_sizesz#ParallelDims.non_data_parallel_sizeN)__name__ __module__ __qualname__int__annotations__boolrrr.propertyr0r%r&r'r3r4r5rr6rrrrrs6 ,       rN) dataclassesr functoolsrtorch.distributed.device_meshr%cosmos_predict2._src.imaginaire.utilsrrrrrrs