#!/bin/bash
# Scaled LLM merge experiment on an L40S (48 GB) — the "firm up the sign" run.
# Prereqs (do ONCE on the login node, which has internet — see hpc/README.md):
#   1) build the env:            uv sync --extra dev --extra neural --extra llm
#      (if the L40S driver is older than CUDA 13, first pin torch — see README)
#   2) pre-download the base:    HF_HOME=$HOME/hf_cache uv run python -c \
#         "from transformers import AutoModelForCausalLM,AutoTokenizer as T; \
#          [f('Qwen/Qwen2.5-7B-Instruct') for f in (T.from_pretrained, AutoModelForCausalLM.from_pretrained)]"
#   submit:  qsub hpc/llm_merge.pbs
#PBS -l select=1:ncpus=8:mem=64gb:ngpus=1:gpu_type=L40S
#PBS -l walltime=04:00:00
#PBS -N lamarckian_llm_merge

cd "$PBS_O_WORKDIR"
export HF_HOME="$HOME/hf_cache"           # models cached here (pre-downloaded on the login node)
export HF_HUB_OFFLINE=1                    # compute node has no internet -> use the cache
                                          # (delete this line if the probe shows internet works)
export TOKENIZERS_PARALLELISM=false

source .venv/bin/activate                 # uv-built env (Python 3.14 + torch + transformers/peft)
nvidia-smi --query-gpu=name,memory.total --format=csv,noheader

python -m llm.experiment configs/llm/merge_hpc.yaml

# results/llm_merge_hpc/{results.parquet,resolved_config.yaml,manifest.json} are written in-place.
# The parquet is gitignored; sync it back to analyse/plot locally, e.g. from your laptop:
#   rsync -avz <user>@login.hpc.ic.ac.uk:'~/…/results/llm_merge_hpc/' results/llm_merge_hpc/
echo "done: $(date)"