fix: isolate per-camera tracker states in tensor-batched execution and add test_tracking

This commit is contained in:
andrew committed 2026-08-19 15:36:46 +07:00
1 parent 2205330679
commit f5c6cff75e
4 files changed
+135 -141

No files matched your search

+4
View File
@@ -5,6 +5,10 @@ All notable changes to the `chicken-counting-sukawarna-det` project are document
## [Unreleased] - 2026-08-19 ## [Unreleased] - 2026-08-19
### 🐛 Bug Fixes ### 🐛 Bug Fixes
- **Multi-Camera Tracker State Isolation (`tracking.py` & `batch_runner.py`)**:
- Implemented per-camera independent tracker instances keyed by `stream_id` in `DetectionTracker`.
- Fixed tracker state bleed and track ID jumping in `tensor_batching` and multi-camera batch modes when camera frames are processed or when active camera sets shrink.
- Added unit test suite `tests/test_tracking.py` covering multi-stream tracker isolation.
- **Multi-Floor Script Syntax (`run_all_coops.sh`)**: - **Multi-Floor Script Syntax (`run_all_coops.sh`)**:
- Resolved fatal `syntax error: unexpected end of file` caused by missing `done` in floor configuration discovery loop. - Resolved fatal `syntax error: unexpected end of file` caused by missing `done` in floor configuration discovery loop.
- Expanded config discovery pattern from `K*-L*.yaml` to `*.yaml` to support custom named coops (e.g. `kandang-atas.yaml`). - Expanded config discovery pattern from `K*-L*.yaml` to `*.yaml` to support custom named coops (e.g. `kandang-atas.yaml`).
+8 -2
View File
@@ -193,7 +193,10 @@ def run_tensor_batched_daily_batch(
break break
if frame_index % inference_stride == 0 or any(not last_batch_tracks[active_cams[i][0]] for i in current_active): if frame_index % inference_stride == 0 or any(not last_batch_tracks[active_cams[i][0]] for i in current_active):
batch_tracks_list = shared_tracker.infer_batch(current_frames, crop_rects=current_crops) current_stream_ids = [active_cams[idx][0] for idx in current_active]
batch_tracks_list = shared_tracker.infer_batch(
current_frames, stream_ids=current_stream_ids, crop_rects=current_crops
)
for i, idx in enumerate(current_active): for i, idx in enumerate(current_active):
cam_id = active_cams[idx][0] cam_id = active_cams[idx][0]
last_batch_tracks[cam_id] = batch_tracks_list[i] last_batch_tracks[cam_id] = batch_tracks_list[i]
@@ -455,7 +458,10 @@ def run_hybrid_daily_batch(
# 2. Batched GPU inference (Synchronous on main thread) # 2. Batched GPU inference (Synchronous on main thread)
if frame_index % inference_stride == 0 or any(not last_batch_tracks[active_cams[i][0]] for i in current_active_now): if frame_index % inference_stride == 0 or any(not last_batch_tracks[active_cams[i][0]] for i in current_active_now):
batch_tracks_list = shared_tracker.infer_batch(current_frames, crop_rects=current_crops) current_stream_ids = [active_cams[idx][0] for idx in current_active_now]
batch_tracks_list = shared_tracker.infer_batch(
current_frames, stream_ids=current_stream_ids, crop_rects=current_crops
)
for i, idx in enumerate(current_active_now): for i, idx in enumerate(current_active_now):
cam_id = active_cams[idx][0] cam_id = active_cams[idx][0]
last_batch_tracks[cam_id] = batch_tracks_list[i] last_batch_tracks[cam_id] = batch_tracks_list[i]
+87 -139
View File
@@ -1,12 +1,16 @@
"""Run YOLO detection and BoT-SORT tracking on each frame.""" """Run YOLO detection and BoT-SORT tracking on each frame with per-camera tracker isolation."""
from __future__ import annotations from __future__ import annotations
import time import time
from pathlib import Path from pathlib import Path
from typing import Any
import numpy as np import numpy as np
import yaml
from ultralytics import YOLO from ultralytics import YOLO
from ultralytics.trackers.track import TRACKER_MAP
from ultralytics.utils import IterableSimpleNamespace
from chicken_counter.config import CameraConfig from chicken_counter.config import CameraConfig
from chicken_counter.engine_utils import ensure_compatible_model from chicken_counter.engine_utils import ensure_compatible_model
@@ -32,6 +36,10 @@ class DetectionTracker:
self.tracker_config_path = str(Path(config.tracker.tracker_config_path)) self.tracker_config_path = str(Path(config.tracker.tracker_config_path))
self.verbose = config.performance.verbose self.verbose = config.performance.verbose
self._infer_count = 0 self._infer_count = 0
self._stream_trackers: dict[str, Any] = {}
self._tracker_cfg_obj: IterableSimpleNamespace | None = None
self._init_tracker_cfg()
print( print(
f"[model] loaded {self.model_kind} from {model_path} " f"[model] loaded {self.model_kind} from {model_path} "
f"(imgsz={config.detection.imgsz}, device={config.detection.device})" f"(imgsz={config.detection.imgsz}, device={config.detection.device})"
@@ -39,7 +47,49 @@ class DetectionTracker:
if self.model_kind == "engine": if self.model_kind == "engine":
print("[model] TensorRT engine active; runtime half flag is ignored") print("[model] TensorRT engine active; runtime half flag is ignored")
def reset_tracking(self) -> None: def _init_tracker_cfg(self) -> None:
try:
tracker_file = Path(self.tracker_config_path)
if tracker_file.exists():
with open(tracker_file, "r", encoding="utf-8") as f:
raw_cfg = yaml.safe_load(f) or {}
self._tracker_cfg_obj = IterableSimpleNamespace(**raw_cfg)
self._tracker_cfg_obj.device = self.config.detection.device
except Exception as exc:
if self.verbose:
print(f"[tracker] error reading tracker config: {exc}")
self._tracker_cfg_obj = None
def get_or_create_tracker(self, stream_id: str = "default") -> Any:
if stream_id not in self._stream_trackers:
if self._tracker_cfg_obj is not None and self._tracker_cfg_obj.tracker_type in TRACKER_MAP:
tracker_cls = TRACKER_MAP[self._tracker_cfg_obj.tracker_type]
self._stream_trackers[stream_id] = tracker_cls(args=self._tracker_cfg_obj)
else:
default_args = IterableSimpleNamespace(
tracker_type="botsort",
track_high_thresh=0.5,
track_low_thresh=0.1,
new_track_thresh=0.6,
track_buffer=self.config.tracker.track_buffer,
match_thresh=0.8,
fuse_score=True,
gmc_method="none",
proximity_thresh=0.5,
appearance_thresh=0.25,
with_reid=False,
model="auto",
device=self.config.detection.device,
)
self._stream_trackers[stream_id] = TRACKER_MAP["botsort"](args=default_args)
return self._stream_trackers[stream_id]
def reset_tracking(self, stream_id: str | None = None) -> None:
if stream_id:
if stream_id in self._stream_trackers:
del self._stream_trackers[stream_id]
else:
self._stream_trackers.clear()
if hasattr(self.model, "predictor"): if hasattr(self.model, "predictor"):
self.model.predictor = None self.model.predictor = None
@@ -47,119 +97,25 @@ class DetectionTracker:
self, self,
frame: np.ndarray, frame: np.ndarray,
*, *,
stream_id: str = "default",
crop_rect: tuple[int, int, int, int] | None = None, crop_rect: tuple[int, int, int, int] | None = None,
) -> list[TrackObservation]: ) -> list[TrackObservation]:
offset_x = 0 results = self.infer_batch([frame], stream_ids=[stream_id], crop_rects=[crop_rect])
offset_y = 0 return results[0] if results else []
source = frame
if crop_rect is not None:
x1, y1, x2, y2 = crop_rect
source = frame[y1:y2, x1:x2]
offset_x, offset_y = x1, y1
track_kwargs: dict = {
"source": source,
"persist": self.config.tracker.persist,
"tracker": self.tracker_config_path,
"conf": self.config.detection.conf,
"iou": self.config.detection.iou,
"classes": self.config.detection.classes,
"imgsz": self.config.detection.imgsz,
"verbose": False,
"device": self.config.detection.device,
}
if self.model_kind != "engine" and self.config.performance.half:
track_kwargs["half"] = True
if self.verbose:
t_start = time.monotonic()
results = self.model.track(**track_kwargs)
if self.verbose:
t_track = time.monotonic()
self._infer_count += 1
if not results:
if self.verbose:
print(f"[tracker #{self._infer_count}] no detections (infer={t_track - t_start:.1f}ms)")
return []
result = results[0]
boxes = result.boxes
if boxes is None or boxes.id is None:
return []
ids = boxes.id.int().cpu().numpy()
classes = boxes.cls.int().cpu().numpy()
confidences = boxes.conf.cpu().numpy()
xyxy = boxes.xyxy.int().cpu().numpy()
mask_polygons = None
if result.masks is not None and result.masks.xy is not None:
mask_polygons = result.masks.xy
if len(mask_polygons) != len(boxes):
raise RuntimeError(
f"Ultralytics box/mask count mismatch: {len(boxes)} boxes, "
f"{len(mask_polygons)} masks"
)
tracks: list[TrackObservation] = []
for index in range(len(boxes)):
track_id = int(ids[index])
class_id = int(classes[index])
confidence = float(confidences[index])
bbox = xyxy[index]
x1 = int(bbox[0]) + offset_x
y1 = int(bbox[1]) + offset_y
x2 = int(bbox[2]) + offset_x
y2 = int(bbox[3]) + offset_y
centroid = ((x1 + x2) // 2, (y1 + y2) // 2)
if crop_rect is not None and not self._centroid_in_rect(centroid, crop_rect):
continue
polygon = None
if mask_polygons is not None:
poly = np.asarray(mask_polygons[index], dtype=np.float64).copy()
if poly.ndim == 2 and poly.shape[0] >= 3:
poly[:, 0] += offset_x
poly[:, 1] += offset_y
polygon = poly
tracks.append(
TrackObservation(
track_id=track_id,
class_id=class_id,
confidence=confidence,
bbox_xyxy=(x1, y1, x2, y2),
centroid=centroid,
mask_polygon_xy=polygon,
)
)
if self.verbose:
unique_ids = sorted(set(t.track_id for t in tracks))
confs = [t.confidence for t in tracks] if tracks else [0]
print(
f"[tracker #{self._infer_count}] "
f"det={len(tracks)} unique={len(unique_ids)} "
f"conf=[{min(confs):.2f}..{max(confs):.2f}] "
f"ids={unique_ids[:10]}{'+' if len(unique_ids) > 10 else ''} "
f"infer={t_track - t_start:.1f}ms"
)
return tracks
def infer_batch( def infer_batch(
self, self,
frames: list[np.ndarray], frames: list[np.ndarray],
*, *,
stream_ids: list[str] | None = None,
crop_rects: list[tuple[int, int, int, int] | None] | None = None, crop_rects: list[tuple[int, int, int, int] | None] | None = None,
) -> list[list[TrackObservation]]: ) -> list[list[TrackObservation]]:
if not frames: if not frames:
return [] return []
if stream_ids is None:
stream_ids = [f"cam_{i}" for i in range(len(frames))]
sources = [] sources = []
offsets = [] offsets = []
for i, frame in enumerate(frames): for i, frame in enumerate(frames):
@@ -172,10 +128,7 @@ class DetectionTracker:
sources.append(frame) sources.append(frame)
offsets.append((0, 0)) offsets.append((0, 0))
track_kwargs: dict = { predict_kwargs: dict = {
"source": sources,
"persist": self.config.tracker.persist,
"tracker": self.tracker_config_path,
"conf": self.config.detection.conf, "conf": self.config.detection.conf,
"iou": self.config.detection.iou, "iou": self.config.detection.iou,
"classes": self.config.detection.classes, "classes": self.config.detection.classes,
@@ -184,19 +137,19 @@ class DetectionTracker:
"device": self.config.detection.device, "device": self.config.detection.device,
} }
if self.model_kind != "engine" and self.config.performance.half: if self.model_kind != "engine" and self.config.performance.half:
track_kwargs["half"] = True predict_kwargs["half"] = True
if self.verbose: if self.verbose:
t_start = time.monotonic() t_start = time.monotonic()
try: try:
results = self.model.track(**track_kwargs) results = self.model.predict(source=sources, **predict_kwargs)
except Exception: except Exception:
results = [] results = []
for src in sources: for src in sources:
kw = dict(track_kwargs) kw = dict(predict_kwargs)
kw["source"] = src kw["source"] = src
res = self.model.track(**kw) res = self.model.predict(**kw)
if res: if res:
results.append(res[0]) results.append(res[0])
@@ -206,53 +159,48 @@ class DetectionTracker:
batch_tracks: list[list[TrackObservation]] = [] batch_tracks: list[list[TrackObservation]] = []
for idx, result in enumerate(results): for idx, result in enumerate(results):
stream_id = stream_ids[idx] if idx < len(stream_ids) else f"stream_{idx}"
tracker = self.get_or_create_tracker(stream_id)
orig_src = sources[idx]
offset_x, offset_y = offsets[idx] offset_x, offset_y = offsets[idx]
crop_rect = crop_rects[idx] if crop_rects and idx < len(crop_rects) else None crop_rect = crop_rects[idx] if crop_rects and idx < len(crop_rects) else None
boxes = result.boxes boxes = result.boxes
if boxes is None or boxes.id is None: if boxes is None or len(boxes) == 0:
tracker.update(np.empty((0, 6)), orig_src)
batch_tracks.append([]) batch_tracks.append([])
continue continue
ids = boxes.id.int().cpu().numpy() det_np = boxes.cpu().numpy()
classes = boxes.cls.int().cpu().numpy() tracks_raw = tracker.update(det_np, orig_src)
confidences = boxes.conf.cpu().numpy()
xyxy = boxes.xyxy.int().cpu().numpy()
mask_polygons = None if len(tracks_raw) == 0:
if result.masks is not None and result.masks.xy is not None: batch_tracks.append([])
mask_polygons = result.masks.xy continue
tracks: list[TrackObservation] = [] tracks: list[TrackObservation] = []
for index in range(len(boxes)): for row in tracks_raw:
track_id = int(ids[index]) # Format: [x1, y1, x2, y2, track_id, conf, cls, idx]
class_id = int(classes[index]) x1 = int(row[0]) + offset_x
confidence = float(confidences[index]) y1 = int(row[1]) + offset_y
bbox = xyxy[index] x2 = int(row[2]) + offset_x
x1 = int(bbox[0]) + offset_x y2 = int(row[3]) + offset_y
y1 = int(bbox[1]) + offset_y track_id = int(row[4])
x2 = int(bbox[2]) + offset_x conf = float(row[5]) if len(row) > 5 else float(self.config.detection.conf)
y2 = int(bbox[3]) + offset_y cls_id = int(row[6]) if len(row) > 6 else 0
centroid = ((x1 + x2) // 2, (y1 + y2) // 2) centroid = ((x1 + x2) // 2, (y1 + y2) // 2)
if crop_rect is not None and not self._centroid_in_rect(centroid, crop_rect): if crop_rect is not None and not self._centroid_in_rect(centroid, crop_rect):
continue continue
polygon = None
if mask_polygons is not None:
poly = np.asarray(mask_polygons[index], dtype=np.float64).copy()
if poly.ndim == 2 and poly.shape[0] >= 3:
poly[:, 0] += offset_x
poly[:, 1] += offset_y
polygon = poly
tracks.append( tracks.append(
TrackObservation( TrackObservation(
track_id=track_id, track_id=track_id,
class_id=class_id, class_id=cls_id,
confidence=confidence, confidence=conf,
bbox_xyxy=(x1, y1, x2, y2), bbox_xyxy=(x1, y1, x2, y2),
centroid=centroid, centroid=centroid,
mask_polygon_xy=polygon, mask_polygon_xy=None,
) )
) )
batch_tracks.append(tracks) batch_tracks.append(tracks)
+36
View File
@@ -0,0 +1,36 @@
"""Tests for DetectionTracker multi-camera stream isolation."""
from __future__ import annotations
import unittest
from pathlib import Path
from unittest.mock import MagicMock, patch
from chicken_counter.config import load_camera_config
class MultiCameraTrackerIsolationTests(unittest.TestCase):
def test_independent_stream_trackers(self) -> None:
cfg = load_camera_config("configs/cameras/example_camera.yaml")
# Test get_or_create_tracker instantiates isolated instances for different stream IDs
with patch("chicken_counter.tracking.ensure_compatible_model", return_value=cfg.detection.model_path), \
patch("chicken_counter.tracking.YOLO"):
from chicken_counter.tracking import DetectionTracker
tracker_manager = DetectionTracker(cfg)
tr1 = tracker_manager.get_or_create_tracker("CC1")
tr2 = tracker_manager.get_or_create_tracker("CC2")
self.assertIsNotNone(tr1)
self.assertIsNotNone(tr2)
self.assertIsNot(tr1, tr2, "Trackers for CC1 and CC2 must be distinct objects")
# Resetting CC1 should not delete CC2 tracker
tracker_manager.reset_tracking("CC1")
self.assertNotIn("CC1", tracker_manager._stream_trackers)
self.assertIn("CC2", tracker_manager._stream_trackers)
if __name__ == "__main__":
unittest.main()