Unify analysis pipeline around the TSV; move tracked DBs out of cloud sync

- Tracked DBs now live at /mnt/data/projects/cupido/tracked/ (out of
  ownCloud to avoid sync conflicts and bandwidth churn). config.py
  TRACKING_OUTPUT_DIR points there; the docker-compose for ethoscope-lab
  mounts it world-readable for JupyterHub users.
- New scripts/export_video_db_index.py joins all_video_info_merged.xlsx
  with the video inventory and the on-disk DBs, producing a TSV that has
  one row per fly/ROI plus training/testing video and DB paths. Handles
  approximate xlsx times, cross-day training/testing, the 12 AM/PM
  ambiguity, and date typos.
- scripts/load_roi_data.py rewritten as a TSV-driven loader returning a
  single DataFrame with session and metadata columns. calculate_distances
  and the two flies_analysis notebooks migrated to use it; downstream
  trained/naive splits remain available via simple equality filters.
- Metadata vocabulary canonicalized: {naïve, niave, untrained, test} all
  resolve to {trained, naive}. Normalization happens at the TSV-export
  boundary (idempotent); the xlsx and the 2025-07-15 legacy CSV were
  edited in place to remove the worst variants.
- scripts/monitor_tracking.py rate calculation fixed: with N parallel
  workers, completions arrive in bursts; the old formula divided by burst
  width and reported nonsense rates. Now uses a 6 h window denominator.
- scripts/track_videos.py: BGRMovieCamera retries cv2.read on transient
  NFS hiccups and a post-tracking completeness gate (≥ 90 % of expected
  duration via MAX(t) across all 6 ROIs) deletes silent partial DBs.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

notebooks/flies_analysis.ipynb

@@ -28,7 +28,22 @@
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": "def load_roi_data():\n    \"\"\"Load ROI data from SQLite databases and group by trained/untrained\"\"\"\n    metadata = pd.read_csv(DATA_METADATA / '2025_07_15_metadata_fixed.csv')\n    metadata['machine_name'] = metadata['machine_name'].astype(str)\n    \n    trained_rois = metadata[metadata['group'] == 'trained']\n    untrained_rois = metadata[metadata['group'] == 'untrained']\n    \n    db_files = list(DATA_RAW.glob('*_tracking.db'))\n    \n    trained_df = pd.DataFrame()\n    untrained_df = pd.DataFrame()\n    \n    for db_file in db_files:\n        print(f\"Processing {db_file.name}\")\n        \n        pattern = r'_([0-9a-f]{32})__'\n        match = re.search(pattern, db_file.name)\n        \n        if not match:\n            print(f\"Could not extract UUID from {db_file.name}\")\n            continue\n            \n        uuid = match.group(1)\n        metadata_matches = metadata[metadata['path'].str.contains(uuid, na=False)]\n        \n        if metadata_matches.empty:\n            print(f\"No metadata matches found for UUID {uuid}\")\n            continue\n            \n        machine_id = metadata_matches.iloc[0]['machine_name']\n        print(f\"Matched to machine ID: {machine_id}\")\n        \n        conn = sqlite3.connect(str(db_file))\n        \n        machine_trained = trained_rois[trained_rois['machine_name'] == machine_id]\n        machine_untrained = untrained_rois[untrained_rois['machine_name'] == machine_id]\n        \n        for _, row in machine_trained.iterrows():\n            roi = row['ROI']\n            try:\n                roi_data = pd.read_sql_query(f\"SELECT * FROM ROI_{roi}\", conn)\n                roi_data['machine_name'] = machine_id\n                roi_data['ROI'] = roi\n                roi_data['group'] = 'trained'\n                trained_df = pd.concat([trained_df, roi_data], ignore_index=True)\n            except Exception as e:\n                print(f\"Error loading ROI_{roi}: {e}\")\n        \n        for _, row in machine_untrained.iterrows():\n            roi = row['ROI']\n            try:\n                roi_data = pd.read_sql_query(f\"SELECT * FROM ROI_{roi}\", conn)\n                roi_data['machine_name'] = machine_id\n                roi_data['ROI'] = roi\n                roi_data['group'] = 'untrained'\n                untrained_df = pd.concat([untrained_df, roi_data], ignore_index=True)\n            except Exception as e:\n                print(f\"Error loading ROI_{roi}: {e}\")\n        \n        conn.close()\n    \n    return trained_df, untrained_df\n\ntrained_data, untrained_data = load_roi_data()\nprint(f\"Trained data shape: {trained_data.shape}\")\nprint(f\"Untrained data shape: {untrained_data.shape}\")\n\ntrained_data.to_csv(DATA_PROCESSED / 'trained_roi_data.csv', index=False)\nuntrained_data.to_csv(DATA_PROCESSED / 'untrained_roi_data.csv', index=False)\nprint(\"Data saved to CSV files\")"
+   "source": [
+    "# Load tracking data via the unified loader (driven by all_video_info_merged.tsv).\n",
+    "# Reason: replaces the old data/raw + 2025_07_15_metadata_fixed.csv path with\n",
+    "# the TSV-based loader that covers the entire batch (2025-07-15 + 2024).\n",
+    "sys.path.insert(0, str(PROJECT_ROOT / 'scripts'))\n",
+    "from load_roi_data import load_roi_data\n",
+    "\n",
+    "data = load_roi_data()\n",
+    "# Backwards-compat slices for the rest of the notebook.\n",
+    "trained_data   = data[data['male'] == 'trained'].copy()\n",
+    "untrained_data = data[data['male'] == 'naive'].copy()\n",
+    "\n",
+    "print(f\"all data:  {data.shape}\")\n",
+    "print(f\"trained:   {trained_data.shape}\")\n",
+    "print(f\"naive:     {untrained_data.shape}\")\n"
+   ]
   },
   {
    "cell_type": "markdown",
@@ -219,4 +234,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

notebooks/flies_analysis_simple.ipynb

@@ -28,7 +28,22 @@
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": "# Load the pre-processed data\ntrained_data = pd.read_csv(DATA_PROCESSED / 'trained_roi_data.csv')\nuntrained_data = pd.read_csv(DATA_PROCESSED / 'untrained_roi_data.csv')\n\nprint(f\"Trained data shape: {trained_data.shape}\")\nprint(f\"Untrained data shape: {untrained_data.shape}\")\nprint(f\"Trained data columns: {list(trained_data.columns)}\")\nprint(f\"Untrained data columns: {list(untrained_data.columns)}\")"
+   "source": [
+    "# Load tracking data via the unified loader (driven by all_video_info_merged.tsv).\n",
+    "# Reason: replaces reads of trained_roi_data.csv / untrained_roi_data.csv with\n",
+    "# the live loader so the notebook always sees the current batch.\n",
+    "sys.path.insert(0, str(PROJECT_ROOT / 'scripts'))\n",
+    "from load_roi_data import load_roi_data\n",
+    "\n",
+    "data = load_roi_data()\n",
+    "trained_data   = data[data['male'] == 'trained'].copy()\n",
+    "untrained_data = data[data['male'] == 'naive'].copy()\n",
+    "\n",
+    "print(f\"all data shape:    {data.shape}\")\n",
+    "print(f\"Trained data:      {trained_data.shape}\")\n",
+    "print(f\"Naive data:        {untrained_data.shape}\")\n",
+    "print(f\"Columns:           {list(trained_data.columns)}\")\n"
+   ]
   },
   {
    "cell_type": "markdown",
@@ -418,4 +433,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}