#!/bin/bash
# Second base lineage (manuscript revision 2026-09-12): merge_seeds (Fisher-Muller, 5 seeds) and
# moe_hard_seeds (headroom rule, 3 seeds) on SmolLM2-1.7B-Instruct. Pre-download the model on the login
# node first:  HF_HOME=$EPHEMERAL/hf_cache uv run python -c "from huggingface_hub import
# snapshot_download; snapshot_download('HuggingFaceTB/SmolLM2-1.7B-Instruct')"
#   submit: qsub hpc/llm_smol.pbs
#PBS -l select=1:ncpus=8:mem=64gb:ngpus=1:gpu_type=L40S
#PBS -l walltime=02:30:00
#PBS -N lam_smol

cd "$PBS_O_WORKDIR"
export HF_HOME="$EPHEMERAL/hf_cache"
export HF_DATASETS_CACHE="$EPHEMERAL/hf_cache/datasets"
export TOKENIZERS_PARALLELISM=false
export UV_CACHE_DIR="$EPHEMERAL/uvcache"
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
source .venv/bin/activate
nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader
mkdir -p configs/llm/_gen
echo "start=$(date)"
for NAME in merge_seeds_smol moe_hard_seeds_smol; do
  echo "== $NAME $(date)"
  python -m llm.experiment "configs/llm/${NAME}.yaml"
done
echo "done: $(date)"