# LBM/buffer will take the configuration file from "configs/LBM" and read the "bufer" entry
python externals/vidar/scripts/datasets/webdataset/create.py $2     \
    --cfg_path utils/webdataset/configs                             \
    --buffer $4                                                     \
    --zone west                                                     \
    --suffix $3                                                     \
    --num_proc $1                                                   \

# Datasets are stored in s3://tri-ml-datasets/cv_webdatasets 
#     
# The buffer key works like this: X_sY
#
# X -> number of context frames each sample should have
# s -> p is with padding, f is without padding 
# Y -> How many valid samples are stored in each shard 
#
# So, 120_p16 means that:
# 
# 1. Each sample has 121 frames (1 current + 120 context), 
# 2. Each shard has 16 valid samples, so it has 120 + 16 = 136 frames
# 3. Final frames are repeated as padding
# 
# FYI: Multiple webdatasets can be created by changing this script, 
# they will be stored in different locations within the same s3 bucket