#!/bin/bash
# The curriculum society campaign (prereg tasks/prereg-llm-society-v4.md, v5 families): one (seed, arm)
# per array element, seeds 2-3 x 4 arms = 8 elements; seed 1 runs locally as the hedge. Baselines run
# once per seed in the `isolated` element (they are cheap and need no partner). Each element checkpoints
# every generation and resumes if requeued.
#   submit:  qsub hpc/llm_curriculum.pbs        status: qstat -u $USER -t
#   index -> seed = 2 + (i-1)/4, arm = (isolated society society_dry seed_bank)[(i-1)%4]
#PBS -l select=1:ncpus=8:mem=64gb:ngpus=1:gpu_type=L40S
#PBS -l walltime=06:00:00
#PBS -N lam_curriculum
#PBS -J 1-8

cd "$PBS_O_WORKDIR"
export HF_HOME="$EPHEMERAL/hf_cache"
export HF_DATASETS_CACHE="$EPHEMERAL/hf_cache/datasets"
export TOKENIZERS_PARALLELISM=false
export UV_CACHE_DIR="$EPHEMERAL/uvcache"
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True

ARMS=(isolated society society_dry seed_bank)
I=$((PBS_ARRAY_INDEX - 1))
SEED=$((2 + I / 4))
ARM=${ARMS[$((I % 4))]}

source .venv/bin/activate
nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader
echo "seed=$SEED arm=$ARM start=$(date)"

CFG="configs/llm/_gen/curriculum_v5_s${SEED}_${ARM}.yaml"
mkdir -p configs/llm/_gen
python - "$SEED" "$ARM" "$CFG" <<'EOF'
import sys, yaml
seed, arm, out = int(sys.argv[1]), sys.argv[2], sys.argv[3]
cfg = yaml.safe_load(open("configs/llm/curriculum_v5_s1.yaml"))
cfg["seed"] = seed
cfg["arms"] = [arm]
cfg["baselines"] = ["sequential", "single_shot_merge", "joint"] if arm == "isolated" else []
cfg["batch_size"] = 48                       # L40S: 46 GB
cfg["train_batch_size"] = 4
cfg["output"] = {"dir": f"results/llm_curriculum_v5/s{seed}_{arm}"}
yaml.safe_dump(cfg, open(out, "w"), sort_keys=False)
EOF

python -m llm.experiment "$CFG"
echo "done: $(date)"