# EXAMPLE: 
# bash custom/webdataset/webdatasetize.sh 8 DDAD/IKED1 40s1 frames 10/0
# 8 -> how many GPUs
# DDAD/IKED1 -> which dataset / which split from its config file
# 40s1 -> webdataset parameters: 41 timesteps, with stride of 1 
# 10/0 -> Split dataset in 10 parts, and process the first (useful for parallel processing)

# Pick a random port in the dynamic/private range (49152–65535)
MASTER_PORT=$(( ( RANDOM % 16384 ) + 49152 ))

echo "Choosing random master port: $MASTER_PORT"

torchrun --nproc-per-node=$1 \
    --master-port=$MASTER_PORT \
    externals/vidar/scripts/datasets/webdataset/webdatasetize.py \
    --datasets=$2 \
    --buffer=$3 \
    --mode=$4 \
    --check_resolution=$5 \
    --resize_ratio=$6 \
    --subset=$7 \
    --cfg_path=custom/prepare/webdataset/configs \
    --parallelize=sequence \
    --merge=True