#!/bin/bash
# Seeds 2-3 for the three single-seed 7B runs (manuscript review 2026-09-11): llm_merge_hpc
# (Fisher-Muller), llm_moe_hard_hpc (union vs fusion, hard) and llm_directed_hard_hpc (directed
# selection, hard). One element per seed; the hard pair stays sequential because directed reuses the
# hard specialists moe trains (models/llm/spec_*_hard_s{seed}). Seed 1 took 8 + 24 min on one L40S.
# Output lands in results/llm_<name>_hpc/s{seed}/ (seed 1 was moved to s1/).
#   submit: qsub hpc/llm_7b_seeds.pbs     status: qstat -u $USER -t
#PBS -l select=1:ncpus=8:mem=64gb:ngpus=1:gpu_type=L40S
#PBS -l walltime=01:30:00
#PBS -N lam_7b_seeds
#PBS -J 2-3

cd "$PBS_O_WORKDIR"
export HF_HOME="$EPHEMERAL/hf_cache"
export TOKENIZERS_PARALLELISM=false
export UV_CACHE_DIR="$EPHEMERAL/uvcache"
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True

SEED=$PBS_ARRAY_INDEX
source .venv/bin/activate
nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader
echo "seed=$SEED start=$(date)"
mkdir -p configs/llm/_gen

for NAME in merge moe_hard directed_hard; do
  CFG="configs/llm/_gen/${NAME}_hpc_s${SEED}.yaml"
  python - "$SEED" "$NAME" "$CFG" <<'PYEOF'
import sys, yaml
seed, name, out = int(sys.argv[1]), sys.argv[2], sys.argv[3]
cfg = yaml.safe_load(open(f"configs/llm/{name}_hpc.yaml"))
cfg["seed"] = seed
cfg["output"] = {"dir": f"results/llm_{name}_hpc/s{seed}"}
yaml.safe_dump(cfg, open(out, "w"), sort_keys=False)
PYEOF
  echo "== $NAME seed=$SEED $(date)"
  python -m llm.experiment "$CFG"
done
echo "done: $(date)"