#!/bin/bash
# Scaled LLM merge experiment on an L40S (46 GB) — the "firm up the sign" run.
# Confirmed on CX3 (probe 2026-07-05): L40S driver 580 supports CUDA 13.0, so the committed
# torch cu130 env works as-is; compute nodes HAVE internet; $TMPDIR is fast local NVMe.
# Prereqs on the LOGIN node (see hpc/README.md): `uv sync --extra dev --extra neural --extra llm`
# (cache on $EPHEMERAL) + pre-download the base into $HF_HOME below.
#   submit:  qsub hpc/llm_merge.pbs      status: qstat -u $USER
#PBS -l select=1:ncpus=8:mem=64gb:ngpus=1:gpu_type=L40S
#PBS -l walltime=02:00:00
#PBS -N lam_llm_merge

cd "$PBS_O_WORKDIR"
export HF_HOME="$EPHEMERAL/hf_cache"       # models cached on the 11 TB ephemeral store
export TOKENIZERS_PARALLELISM=false
export UV_CACHE_DIR="$EPHEMERAL/uvcache"

source .venv/bin/activate                  # uv-built env (Python 3.14 + torch cu130 + transformers/peft)
nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader

python -m llm.experiment configs/llm/merge_hpc.yaml

# results/llm_merge_hpc/ is written in-place (parquet gitignored). Sync back to plot:
#   rsync -avz hpc:'…/LamarckianAI/results/llm_merge_hpc/' results/llm_merge_hpc/
echo "done: $(date)"