#!/bin/bash
# Reconnaissance job — resolves the two unknowns the RCS docs don't cover:
#   (a) do compute nodes have internet? (b) what CUDA version does the L40S driver support?
# plus TMPDIR/disk and available python/cuda modules. Cheap: 1 GPU, 10 min.
#   submit:  qsub hpc/probe.pbs      status: qstat -u $USER      output: probe.o<jobid>
#PBS -l select=1:ncpus=4:mem=24gb:ngpus=1
#PBS -l walltime=00:10:00
#PBS -N lamarckian_probe

cd "$PBS_O_WORKDIR" || cd "$HOME"
echo "=== node / date ==="; hostname; date
echo; echo "=== GPU ==="; nvidia-smi --query-gpu=name,driver_version,memory.total --format=csv 2>&1
echo "--- max CUDA the driver supports (top-right of nvidia-smi) ---"; nvidia-smi 2>&1 | head -4
echo; echo "=== compute-node internet? (critical) ==="
curl -sS -m 15 -o /dev/null -w "huggingface.co -> HTTP %{http_code}\n" https://huggingface.co 2>&1 \
  || echo ">>> NO internet on compute node (must pre-download models on the login node)"
echo; echo "=== TMPDIR (node-local scratch) ==="; echo "TMPDIR=$TMPDIR"; df -h "$TMPDIR" 2>/dev/null | tail -1
echo "=== \$HOME disk ==="; df -h "$HOME" 2>/dev/null | tail -1
echo; echo "=== python / cuda / conda modules available ==="
module avail 2>&1 | tr ' ' '\n' | grep -iE "^(python|cuda|anaconda|miniforge|gcc)" | sort -u | head -40
echo; echo "=== uv installed? ==="; command -v uv 2>/dev/null || ls -la ~/.local/bin/uv 2>/dev/null || echo "uv not installed (curl -LsSf https://astral.sh/uv/install.sh | sh)"
echo "=== system python3 ==="; python3 --version 2>&1
echo; echo "=== done ==="