Unify analysis pipeline around the TSV; move tracked DBs out of cloud sync
- Tracked DBs now live at /mnt/data/projects/cupido/tracked/ (out of
ownCloud to avoid sync conflicts and bandwidth churn). config.py
TRACKING_OUTPUT_DIR points there; the docker-compose for ethoscope-lab
mounts it world-readable for JupyterHub users.
- New scripts/export_video_db_index.py joins all_video_info_merged.xlsx
with the video inventory and the on-disk DBs, producing a TSV that has
one row per fly/ROI plus training/testing video and DB paths. Handles
approximate xlsx times, cross-day training/testing, the 12 AM/PM
ambiguity, and date typos.
- scripts/load_roi_data.py rewritten as a TSV-driven loader returning a
single DataFrame with session and metadata columns. calculate_distances
and the two flies_analysis notebooks migrated to use it; downstream
trained/naive splits remain available via simple equality filters.
- Metadata vocabulary canonicalized: {naïve, niave, untrained, test} all
resolve to {trained, naive}. Normalization happens at the TSV-export
boundary (idempotent); the xlsx and the 2025-07-15 legacy CSV were
edited in place to remove the worst variants.
- scripts/monitor_tracking.py rate calculation fixed: with N parallel
workers, completions arrive in bursts; the old formula divided by burst
width and reported nonsense rates. Now uses a 6 h window denominator.
- scripts/track_videos.py: BGRMovieCamera retries cv2.read on transient
NFS hiccups and a post-tracking completeness gate (≥ 90 % of expected
duration via MAX(t) across all 6 ROIs) deletes silent partial DBs.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
parent
e4da7691d5
commit
f60a9d0530
13 changed files with 569 additions and 237 deletions
|
|
@ -1,117 +1,99 @@
|
|||
import pandas as pd
|
||||
"""Compute per-frame inter-fly distances for every (date, machine, ROI, session).
|
||||
|
||||
Reads tracking data via :func:`load_roi_data.load_roi_data` (which is driven
|
||||
by ``all_video_info_merged.tsv``) and produces one distances DataFrame
|
||||
spanning every fly/session in the batch. Group membership (``trained`` /
|
||||
``untrained``) is preserved from the ``male`` column.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from scipy.spatial.distance import euclidean
|
||||
|
||||
from config import DATA_PROCESSED
|
||||
from load_roi_data import load_roi_data
|
||||
|
||||
|
||||
def calculate_fly_distances(trained_file=None, untrained_file=None):
|
||||
"""Calculate distances between flies at each time point.
|
||||
def calculate_fly_distances(data: pd.DataFrame | None = None) -> pd.DataFrame:
|
||||
"""Compute inter-fly distances over time for every fly/session.
|
||||
|
||||
For each time point:
|
||||
- If two flies are detected: calculate Cartesian distance between them
|
||||
- If one fly is detected: set distance to 0 if area > average area, otherwise NaN
|
||||
For each time point inside one (date, machine, ROI, session) trajectory:
|
||||
- 2+ flies detected: Euclidean distance between the first two by id
|
||||
- 1 fly detected: distance = 0 if its bbox area exceeds the global
|
||||
mean (likely a single blob containing both flies), else NaN
|
||||
|
||||
Args:
|
||||
trained_file (Path): Path to trained ROI data CSV.
|
||||
untrained_file (Path): Path to untrained ROI data CSV.
|
||||
data: optional pre-loaded DataFrame from :func:`load_roi_data`. If
|
||||
None, the full batch is loaded.
|
||||
|
||||
Returns:
|
||||
tuple: (trained_distances, untrained_distances) DataFrames.
|
||||
DataFrame with one row per (track, time) pair, including ``distance``,
|
||||
``n_flies``, ``area_fly1``, ``area_fly2``, plus the metadata columns
|
||||
propagated from the source row (``date``, ``machine_name``, ``ROI``,
|
||||
``session``, ``male``, ``species``, ``memory``, ``age``).
|
||||
"""
|
||||
if trained_file is None:
|
||||
trained_file = DATA_PROCESSED / 'trained_roi_data.csv'
|
||||
if untrained_file is None:
|
||||
untrained_file = DATA_PROCESSED / 'untrained_roi_data.csv'
|
||||
if data is None:
|
||||
data = load_roi_data()
|
||||
if data.empty:
|
||||
return pd.DataFrame()
|
||||
|
||||
trained_df = pd.read_csv(trained_file)
|
||||
untrained_df = pd.read_csv(untrained_file)
|
||||
|
||||
trained_df['area'] = trained_df['w'] * trained_df['h']
|
||||
untrained_df['area'] = untrained_df['w'] * untrained_df['h']
|
||||
|
||||
avg_area = np.mean([trained_df['area'].mean(), untrained_df['area'].mean()])
|
||||
data = data.copy()
|
||||
data["area"] = data["w"] * data["h"]
|
||||
avg_area = data["area"].mean()
|
||||
print(f"Average area across all data: {avg_area:.2f}")
|
||||
|
||||
trained_distances = process_distance_data(trained_df, avg_area)
|
||||
untrained_distances = process_distance_data(untrained_df, avg_area)
|
||||
# Carry these onto every output row (constant within a track).
|
||||
keep_meta = ["date", "machine_name", "ROI", "session", "male",
|
||||
"species", "memory", "age"]
|
||||
|
||||
return trained_distances, untrained_distances
|
||||
|
||||
|
||||
def process_distance_data(df, avg_area):
|
||||
"""Process a DataFrame to calculate distances between flies at each time point.
|
||||
|
||||
Args:
|
||||
df (pd.DataFrame): Input tracking data.
|
||||
avg_area (float): Average area threshold for single-fly detection.
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: Distance data with columns for machine, ROI, time, distance.
|
||||
"""
|
||||
results = []
|
||||
|
||||
for (machine_name, roi), group in df.groupby(['machine_name', 'ROI']):
|
||||
for t, time_group in group.groupby('t'):
|
||||
time_group = time_group.sort_values('id').reset_index(drop=True)
|
||||
rows: list[dict] = []
|
||||
track_keys = ["date", "machine_name", "ROI", "session"]
|
||||
for track, track_df in data.groupby(track_keys, sort=False):
|
||||
meta_row = {k: v for k, v in zip(track_keys, track)}
|
||||
# Carry the rest of the metadata from any sample (constant per track).
|
||||
sample = track_df.iloc[0]
|
||||
for col in keep_meta:
|
||||
if col not in meta_row:
|
||||
meta_row[col] = sample[col]
|
||||
|
||||
for t, time_group in track_df.groupby("t", sort=False):
|
||||
time_group = time_group.sort_values("id").reset_index(drop=True)
|
||||
row = dict(meta_row)
|
||||
row["t"] = t
|
||||
if len(time_group) >= 2:
|
||||
fly1 = time_group.iloc[0]
|
||||
fly2 = time_group.iloc[1]
|
||||
distance = euclidean([fly1['x'], fly1['y']], [fly2['x'], fly2['y']])
|
||||
f1, f2 = time_group.iloc[0], time_group.iloc[1]
|
||||
row["distance"] = euclidean([f1["x"], f1["y"]], [f2["x"], f2["y"]])
|
||||
row["n_flies"] = len(time_group)
|
||||
row["area_fly1"] = f1["area"]
|
||||
row["area_fly2"] = f2["area"]
|
||||
else:
|
||||
f = time_group.iloc[0]
|
||||
row["distance"] = 0.0 if f["area"] > avg_area else np.nan
|
||||
row["n_flies"] = 1
|
||||
row["area_fly1"] = f["area"]
|
||||
row["area_fly2"] = np.nan
|
||||
rows.append(row)
|
||||
|
||||
results.append({
|
||||
'machine_name': machine_name,
|
||||
'ROI': roi,
|
||||
't': t,
|
||||
'distance': distance,
|
||||
'n_flies': len(time_group),
|
||||
'area_fly1': fly1['area'],
|
||||
'area_fly2': fly2['area']
|
||||
})
|
||||
elif len(time_group) == 1:
|
||||
fly = time_group.iloc[0]
|
||||
area = fly['area']
|
||||
|
||||
if area > avg_area:
|
||||
distance = 0.0
|
||||
else:
|
||||
distance = np.nan
|
||||
|
||||
results.append({
|
||||
'machine_name': machine_name,
|
||||
'ROI': roi,
|
||||
't': t,
|
||||
'distance': distance,
|
||||
'n_flies': 1,
|
||||
'area_fly1': area,
|
||||
'area_fly2': np.nan
|
||||
})
|
||||
|
||||
return pd.DataFrame(results)
|
||||
return pd.DataFrame(rows)
|
||||
|
||||
|
||||
def main():
|
||||
"""Run distance calculations and save results."""
|
||||
trained_distances, untrained_distances = calculate_fly_distances()
|
||||
def main() -> None:
|
||||
distances = calculate_fly_distances()
|
||||
|
||||
print(f"Trained data distance summary:")
|
||||
print(f" Shape: {trained_distances.shape}")
|
||||
print(f" Distance stats:")
|
||||
print(f" Count: {trained_distances['distance'].count()}")
|
||||
print(f" Mean: {trained_distances['distance'].mean():.2f}")
|
||||
print(f" Std: {trained_distances['distance'].std():.2f}")
|
||||
print("\nDistance summary:")
|
||||
print(f" Shape: {distances.shape}")
|
||||
if not distances.empty:
|
||||
print(f" Distance count: {distances['distance'].count()}")
|
||||
print(f" Distance mean: {distances['distance'].mean():.2f}")
|
||||
print(f" Distance std: {distances['distance'].std():.2f}")
|
||||
male = distances["male"]
|
||||
print(f" Trained tracks: {(male == 'trained').sum()}")
|
||||
print(f" Naive tracks: {(male == 'naive').sum()}")
|
||||
|
||||
print(f"\nUntrained data distance summary:")
|
||||
print(f" Shape: {untrained_distances.shape}")
|
||||
print(f" Distance stats:")
|
||||
print(f" Count: {untrained_distances['distance'].count()}")
|
||||
print(f" Mean: {untrained_distances['distance'].mean():.2f}")
|
||||
print(f" Std: {untrained_distances['distance'].std():.2f}")
|
||||
|
||||
trained_distances.to_csv(DATA_PROCESSED / 'trained_distances.csv', index=False)
|
||||
untrained_distances.to_csv(DATA_PROCESSED / 'untrained_distances.csv', index=False)
|
||||
print("\nDistance data saved")
|
||||
DATA_PROCESSED.mkdir(parents=True, exist_ok=True)
|
||||
out = DATA_PROCESSED / "distances.csv"
|
||||
distances.to_csv(out, index=False)
|
||||
print(f"\nSaved {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue