#!/bin/bash
# Curriculum controls (manuscript review 2026-09-11): seeds 2-3 of (a) the forced-stop arm
# `curriculum_v5_stop3` and (b) the decorrelated curriculum `curriculum_v5_decor` (isolated + veto arms
# in one element, ~1 h). Seed 1 of each runs locally. Pairs against the existing v5 seeds 2-3.
#   submit: qsub hpc/llm_curriculum_controls.pbs     status: qstat -u $USER -t
#   index -> seed = 2 + (i-1)/2, config = (stop3 decor)[(i-1)%2]
#PBS -l select=1:ncpus=8:mem=64gb:ngpus=1:gpu_type=L40S
#PBS -l walltime=02:30:00
#PBS -N lam_ctrl
#PBS -J 1-4

cd "$PBS_O_WORKDIR"
export HF_HOME="$EPHEMERAL/hf_cache"
export HF_DATASETS_CACHE="$EPHEMERAL/hf_cache/datasets"
export TOKENIZERS_PARALLELISM=false
export UV_CACHE_DIR="$EPHEMERAL/uvcache"
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True

NAMES=(stop3 decor)
I=$((PBS_ARRAY_INDEX - 1))
SEED=$((2 + I / 2))
NAME=${NAMES[$((I % 2))]}

source .venv/bin/activate
nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader
echo "seed=$SEED config=$NAME start=$(date)"

CFG="configs/llm/_gen/curriculum_v5_${NAME}_s${SEED}.yaml"
mkdir -p configs/llm/_gen
python - "$SEED" "$NAME" "$CFG" <<'PYEOF'
import sys, yaml
seed, name, out = int(sys.argv[1]), sys.argv[2], sys.argv[3]
cfg = yaml.safe_load(open(f"configs/llm/curriculum_v5_{name}.yaml"))
cfg["seed"] = seed
cfg["batch_size"] = 48                       # L40S: 46 GB (matches the v5 seeds 2-3 runs)
cfg["train_batch_size"] = 4
cfg["output"] = {"dir": f"results/llm_curriculum_v5_{name}/s{seed}"}
yaml.safe_dump(cfg, open(out, "w"), sort_keys=False)
PYEOF

python -m llm.experiment "$CFG"
echo "done: $(date)"