#!/bin/bash
# Module-level union-preserving recombination (llm_moe) on an L40S (46 GB) — route/max-merge vs fusion.
# Reuses the specialist adapters trained by the llm_merge_hpc run if models/llm/spec_* is present on
# the node; otherwise trains them fresh. Same env as hpc/llm_merge.pbs (see hpc/README.md):
#   `uv sync --extra dev --extra neural --extra llm` on the login node + pre-download the 7B base.
#   submit:  qsub hpc/llm_moe.pbs      status: qstat -u $USER
#PBS -l select=1:ncpus=8:mem=64gb:ngpus=1:gpu_type=L40S
#PBS -l walltime=02:00:00
#PBS -N lam_llm_moe

cd "$PBS_O_WORKDIR"
export HF_HOME="$EPHEMERAL/hf_cache"
export TOKENIZERS_PARALLELISM=false
export UV_CACHE_DIR="$EPHEMERAL/uvcache"

source .venv/bin/activate
nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader

python -m llm.experiment configs/llm/moe_hpc.yaml

# results/llm_moe_hpc/ written in-place (parquet gitignored). Sync back to plot:
#   rsync -avz hpc:'…/LamarckianAI/results/llm_moe_hpc/' results/llm_moe_hpc/
echo "done: $(date)"