#!/bin/bash
# The LLM society campaign (C3): 4-arm ablation, one seed per array element on one L40S each.
# Seed 1 runs locally on GG's A4000 (queue insurance); this array covers seeds 2-4. Each job is
# self-contained (founders trained inline, adapters under models/, deleted per generation), so a
# killed element reruns cleanly and elements schedule independently. ~2.5-3.5 h measured-scale
# runtime; 6 h walltime for backfill-friendly margin.
#   submit:  qsub hpc/llm_society.pbs      status: qstat -u $USER
#PBS -l select=1:ncpus=8:mem=64gb:ngpus=1:gpu_type=L40S
#PBS -l walltime=06:00:00
#PBS -N lam_society
#PBS -J 2-4

cd "$PBS_O_WORKDIR"
export HF_HOME="$EPHEMERAL/hf_cache"
export TOKENIZERS_PARALLELISM=false
export UV_CACHE_DIR="$EPHEMERAL/uvcache"

source .venv/bin/activate
nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader

python -m llm.experiment "configs/llm/society_campaign_s${PBS_ARRAY_INDEX}.yaml"

# results/llm_society_campaign/s${PBS_ARRAY_INDEX}/ written in-place. Sync back:
#   rsync -avz hpc:'.../results/llm_society_campaign/' results/llm_society_campaign/
echo "done: $(date)"