From 231c7a437fb241ff290c1d8fbcb5c114f0354701 Mon Sep 17 00:00:00 2001 From: Giorgio Gilestro Date: Fri, 1 May 2026 08:55:44 +0100 Subject: [PATCH] Remove hardcoded /home/gg paths so the project is portable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Notebooks now use Path.home() / "cupido" for the repo root (works for any user inside the JupyterLab container), and the offline-tracking scripts read the ethoscope source-tree location from the new ETHOSCOPE_SRC config constant — defaulting to ~/Code/ethoscope_project/... and overridable via the ETHOSCOPE_SRC environment variable. Co-Authored-By: Claude Opus 4.7 --- README.md | 2 +- notebooks/flies_analysis.ipynb | 2 +- notebooks/flies_analysis_simple.ipynb | 2 +- notebooks/getting_started/00_welcome.ipynb | 4 ++-- .../03_compare_trained_vs_naive.ipynb | 2 +- scripts/auto_detect_targets.py | 6 +++--- scripts/config.py | 17 ++++++++++++++--- scripts/track_videos.py | 4 ++-- 8 files changed, 25 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5644fea..6c54237 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ is split in two stages so you can sit at the screen and click for an hour, then let the tracker grind through overnight. ```bash -# extra deps (ethoscope src must be at /home/gg/Code/ethoscope_project/...) +# extra deps (set ETHOSCOPE_SRC env var if your ethoscope clone isn't at ~/Code/ethoscope_project/...) pip install -r requirements-tracking.txt # 1) build the inventory (xlsx ↔ /mnt/ethoscope_data/videos/) diff --git a/notebooks/flies_analysis.ipynb b/notebooks/flies_analysis.ipynb index ab4c127..e832ed5 100644 --- a/notebooks/flies_analysis.ipynb +++ b/notebooks/flies_analysis.ipynb @@ -14,7 +14,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "import sys\nfrom pathlib import Path\n\nimport pandas as pd\nimport numpy as np\nimport sqlite3\nimport glob\nimport re\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy.spatial.distance import euclidean\nfrom scipy import stats\n\n# Two locations to know about:\n# - DATA_DIR : where the project's bulky data lives (data volume)\n# - REPO_ROOT : where the code repository lives (this notebook is inside it)\nDATA_DIR = Path(\"/mnt/data/projects/cupido\")\nREPO_ROOT = Path(\"/home/gg/ownCloud/Work/Projects/coding/cupido/tracking\")\n\n# Pull every other path constant from scripts/config.py so this notebook\n# stays in sync with the rest of the codebase. (Reason: avoids drift when\n# paths change — config.py is the single source of truth.)\nsys.path.insert(0, str(REPO_ROOT / \"scripts\"))\nfrom config import DATA_RAW, DATA_METADATA, DATA_PROCESSED, FIGURES\n\n# Plotting style\nplt.style.use('seaborn-v0_8')\nsns.set_palette(\"husl\")\n" + "source": "import sys\nfrom pathlib import Path\n\nimport pandas as pd\nimport numpy as np\nimport sqlite3\nimport glob\nimport re\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy.spatial.distance import euclidean\nfrom scipy import stats\n\n# Two locations to know about:\n# - DATA_DIR : where the project's bulky data lives (mounted into the container)\n# - REPO_ROOT : where the code repository is checked out (your home directory)\n# Path.home() expands to the current user's home, so this works for any\n# user running the container (no hard-coded usernames).\nDATA_DIR = Path(\"/mnt/data/projects/cupido\")\nREPO_ROOT = Path.home() / \"cupido\"\n\n# Pull every other path constant from scripts/config.py so this notebook\n# stays in sync with the rest of the codebase. (Reason: avoids drift when\n# paths change — config.py is the single source of truth.)\nsys.path.insert(0, str(REPO_ROOT / \"scripts\"))\nfrom config import DATA_RAW, DATA_METADATA, DATA_PROCESSED, FIGURES\n\n# Plotting style\nplt.style.use('seaborn-v0_8')\nsns.set_palette(\"husl\")\n" }, { "cell_type": "markdown", diff --git a/notebooks/flies_analysis_simple.ipynb b/notebooks/flies_analysis_simple.ipynb index 6e1e7f7..f10e9f8 100644 --- a/notebooks/flies_analysis_simple.ipynb +++ b/notebooks/flies_analysis_simple.ipynb @@ -14,7 +14,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "import sys\nfrom pathlib import Path\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy.spatial.distance import euclidean\nfrom scipy import stats\n\n# Two locations to know about:\n# - DATA_DIR : where the project's bulky data lives (data volume)\n# - REPO_ROOT : where the code repository lives (this notebook is inside it)\nDATA_DIR = Path(\"/mnt/data/projects/cupido\")\nREPO_ROOT = Path(\"/home/gg/ownCloud/Work/Projects/coding/cupido/tracking\")\n\n# Pull every other path constant from scripts/config.py so this notebook\n# stays in sync with the rest of the codebase. (Reason: avoids drift when\n# paths change — config.py is the single source of truth.)\nsys.path.insert(0, str(REPO_ROOT / \"scripts\"))\nfrom config import DATA_RAW, DATA_METADATA, DATA_PROCESSED, FIGURES\n\n# Plotting style\nplt.style.use('seaborn-v0_8')\nsns.set_palette(\"husl\")\n\nprint(f\"Data directory: {DATA_DIR}\")\nprint(f\"Repo root: {REPO_ROOT}\")\nprint(f\"Pandas version: {pd.__version__}\")\nprint(f\"NumPy version: {np.__version__}\")\n" + "source": "import sys\nfrom pathlib import Path\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy.spatial.distance import euclidean\nfrom scipy import stats\n\n# Two locations to know about:\n# - DATA_DIR : where the project's bulky data lives (mounted into the container)\n# - REPO_ROOT : where the code repository is checked out (your home directory)\n# Path.home() expands to the current user's home, so this works for any\n# user running the container (no hard-coded usernames).\nDATA_DIR = Path(\"/mnt/data/projects/cupido\")\nREPO_ROOT = Path.home() / \"cupido\"\n\n# Pull every other path constant from scripts/config.py so this notebook\n# stays in sync with the rest of the codebase. (Reason: avoids drift when\n# paths change — config.py is the single source of truth.)\nsys.path.insert(0, str(REPO_ROOT / \"scripts\"))\nfrom config import DATA_RAW, DATA_METADATA, DATA_PROCESSED, FIGURES\n\n# Plotting style\nplt.style.use('seaborn-v0_8')\nsns.set_palette(\"husl\")\n\nprint(f\"Data directory: {DATA_DIR}\")\nprint(f\"Repo root: {REPO_ROOT}\")\nprint(f\"Pandas version: {pd.__version__}\")\nprint(f\"NumPy version: {np.__version__}\")\n" }, { "cell_type": "markdown", diff --git a/notebooks/getting_started/00_welcome.ipynb b/notebooks/getting_started/00_welcome.ipynb index a23adba..144ab89 100644 --- a/notebooks/getting_started/00_welcome.ipynb +++ b/notebooks/getting_started/00_welcome.ipynb @@ -149,14 +149,14 @@ { "cell_type": "markdown", "metadata": {}, - "source": "## Where everything lives\n\nTake a moment to memorize these locations — you'll come back to them often.\n\n| what | where |\n|---|---|\n| Tracking DBs (SQLite, one per video) | `/mnt/data/projects/cupido/tracked/` |\n| Target JSONs (the user-clicked reference points) | `/mnt/data/projects/cupido/targets/` |\n| The metadata table (xlsx + TSV) | `/mnt/data/projects/cupido/all_video_info_merged.tsv` |\n| Source video files | `/mnt/ethoscope_data/videos/` |\n| Project code (this repo) | `/home/gg/ownCloud/Work/Projects/coding/cupido/tracking/` |\n| Your notebooks | `notebooks/getting_started/` (this folder) |\n\nNotice the pattern: **everything bulky or regenerable lives under\n`/mnt/data/projects/cupido/`**. The repository itself only stores code,\ndocumentation, and small metadata files. We'll refer to that data\ndirectory as `DATA_DIR` from here on.\n\nLet's verify a couple of these from inside Python:\n" + "source": "## Where everything lives\n\nTake a moment to memorize these locations — you'll come back to them often.\n\n| what | where |\n|---|---|\n| Tracking DBs (SQLite, one per video) | `/mnt/data/projects/cupido/tracked/` |\n| Target JSONs (the user-clicked reference points) | `/mnt/data/projects/cupido/targets/` |\n| The metadata table (xlsx + TSV) | `/mnt/data/projects/cupido/all_video_info_merged.tsv` |\n| Source video files | `/mnt/ethoscope_data/videos/` |\n| Project code (this repo) | `~/cupido/` (your home directory inside the container) |\n| Your notebooks | `~/cupido/notebooks/getting_started/` (this folder) |\n\nNotice the pattern: **everything bulky or regenerable lives under\n`/mnt/data/projects/cupido/`**. The repository itself, with all the\ncode and notebooks, is checked out into your home directory inside the\nJupyterLab container — that's `~/cupido`, where `~` is shorthand for\nyour home directory (`/home/`).\n\nWe'll refer to those two roots as `DATA_DIR` and `REPO_ROOT` from here on.\n\nLet's verify a couple of these from inside Python:\n" }, { "cell_type": "code", "metadata": {}, "execution_count": null, "outputs": [], - "source": "from pathlib import Path\n\n# Single root for all the bulky / regenerable project data.\nDATA_DIR = Path(\"/mnt/data/projects/cupido\")\n\ntracked_dir = DATA_DIR / \"tracked\"\ntargets_dir = DATA_DIR / \"targets\"\nmetadata_tsv = DATA_DIR / \"all_video_info_merged.tsv\"\n\nprint(f\"Tracking DBs available: {len(list(tracked_dir.glob('*_tracking.db')))}\")\nprint(f\"Target JSONs available: {len(list(targets_dir.glob('*.json')))}\")\nprint(f\"Metadata TSV exists: {metadata_tsv.exists()}\")\n" + "source": "from pathlib import Path\n\n# The two roots we keep coming back to.\nDATA_DIR = Path(\"/mnt/data/projects/cupido\") # bulky data, mounted into the container\nREPO_ROOT = Path.home() / \"cupido\" # the code repo in your home dir\n\ntracked_dir = DATA_DIR / \"tracked\"\ntargets_dir = DATA_DIR / \"targets\"\nmetadata_tsv = DATA_DIR / \"all_video_info_merged.tsv\"\n\nprint(f\"Repo root: {REPO_ROOT} (exists={REPO_ROOT.exists()})\")\nprint(f\"Tracking DBs available: {len(list(tracked_dir.glob('*_tracking.db')))}\")\nprint(f\"Target JSONs available: {len(list(targets_dir.glob('*.json')))}\")\nprint(f\"Metadata TSV exists: {metadata_tsv.exists()}\")\n" }, { "cell_type": "markdown", diff --git a/notebooks/getting_started/03_compare_trained_vs_naive.ipynb b/notebooks/getting_started/03_compare_trained_vs_naive.ipynb index 35d6590..71818f3 100644 --- a/notebooks/getting_started/03_compare_trained_vs_naive.ipynb +++ b/notebooks/getting_started/03_compare_trained_vs_naive.ipynb @@ -48,7 +48,7 @@ "metadata": {}, "execution_count": null, "outputs": [], - "source": "import sys\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom scipy import stats\n\n# Two locations to know about:\n# - DATA_DIR : where the project's data files live (read-only data volume)\n# - REPO_ROOT : where the code repository lives (this notebook is inside it)\n# We build both as Path objects, then derive everything else from them.\nDATA_DIR = Path(\"/mnt/data/projects/cupido\")\nREPO_ROOT = Path(\"/home/gg/ownCloud/Work/Projects/coding/cupido/tracking\")\n\n# Tell Python where to find the project's helper modules (in scripts/).\nsys.path.insert(0, str(REPO_ROOT / \"scripts\"))\n\nfrom load_roi_data import load_roi_data\n" + "source": "import sys\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom scipy import stats\n\n# Two locations to know about:\n# - DATA_DIR : where the project's data files live (mounted into the container)\n# - REPO_ROOT : where the code repository is checked out (your home directory)\n# Path.home() expands to the current user's home directory, so this\n# works for any user running the container — no hard-coded usernames.\nDATA_DIR = Path(\"/mnt/data/projects/cupido\")\nREPO_ROOT = Path.home() / \"cupido\"\n\n# Tell Python where to find the project's helper modules (in scripts/).\nsys.path.insert(0, str(REPO_ROOT / \"scripts\"))\n\nfrom load_roi_data import load_roi_data\n" }, { "cell_type": "markdown", diff --git a/scripts/auto_detect_targets.py b/scripts/auto_detect_targets.py index 077ac41..4107c41 100644 --- a/scripts/auto_detect_targets.py +++ b/scripts/auto_detect_targets.py @@ -24,10 +24,10 @@ import cv2 import numpy as np import pandas as pd -# ethoscope source tree -sys.path.insert(0, "/home/gg/Code/ethoscope_project/ethoscope/src/ethoscope") +from config import ETHOSCOPE_SRC, INVENTORY_CSV, TARGETS_DIR -from config import INVENTORY_CSV, TARGETS_DIR # noqa: E402 +# ethoscope source tree +sys.path.insert(0, str(ETHOSCOPE_SRC)) from ethoscope.roi_builders.target_roi_builder import TargetGridROIBuilder # noqa: E402 diff --git a/scripts/config.py b/scripts/config.py index b0ff682..2e4bda0 100644 --- a/scripts/config.py +++ b/scripts/config.py @@ -1,5 +1,6 @@ """Shared path constants for the Cupido tracking project.""" +import os from pathlib import Path # Where this code repository lives (the directory containing scripts/, notebooks/, ...). @@ -13,9 +14,9 @@ LOGS_DIR = PROJECT_ROOT / "data" / "logs" # Where the source videos live (read-only NFS mount). VIDEOS_ROOT = Path("/mnt/ethoscope_data/videos") -# Where the project's bulky data lives — outside the ownCloud-synced repo so -# it doesn't churn the cloud sync. This single root holds everything that's -# big or regenerable: tracking DBs, target-point JSONs, and the metadata +# Where the project's bulky data lives — outside the cloud-synced repo so +# it doesn't churn the sync. This single root holds everything that's big +# or regenerable: tracking DBs, target-point JSONs, and the metadata # spreadsheet (xlsx + TSV). DATA_VOLUME = Path("/mnt/data/projects/cupido") TARGETS_DIR = DATA_VOLUME / "targets" @@ -25,3 +26,13 @@ VIDEO_INFO_TSV = DATA_VOLUME / "all_video_info_merged.tsv" # A small CSV listing every video file we know about (built locally). INVENTORY_CSV = DATA_METADATA / "video_inventory.csv" + +# Where the ethoscope source tree is checked out (used by track_videos.py +# and auto_detect_targets.py — host-side scripts that import ethoscope +# from a local clone rather than from pip). Default assumes the standard +# layout `~/Code/ethoscope_project/ethoscope/src/ethoscope`; override with +# the ETHOSCOPE_SRC environment variable if your clone lives elsewhere. +ETHOSCOPE_SRC = Path(os.environ.get( + "ETHOSCOPE_SRC", + str(Path.home() / "Code" / "ethoscope_project" / "ethoscope" / "src" / "ethoscope"), +)) diff --git a/scripts/track_videos.py b/scripts/track_videos.py index cb65292..299e4ec 100644 --- a/scripts/track_videos.py +++ b/scripts/track_videos.py @@ -27,11 +27,11 @@ from pathlib import Path import numpy as np +from config import ETHOSCOPE_SRC, TARGETS_DIR, TRACKING_OUTPUT_DIR + # Import ethoscope from the local source tree (no pip install). -ETHOSCOPE_SRC = Path("/home/gg/Code/ethoscope_project/ethoscope/src/ethoscope") sys.path.insert(0, str(ETHOSCOPE_SRC)) -from config import TARGETS_DIR, TRACKING_OUTPUT_DIR # noqa: E402 from tracking_geometry import HD_FG_DATA, compute_roi_polygons # noqa: E402