fix: resolve multi-threading DB concurrency, disk traversal caching, and coops discovery

configs/floor_config/kandang-atas.yaml

@@ -0,0 +1,7 @@
+# Floor Configuration for kandang-atas
+# Inherits all default settings, tracker options, model path, and stage thresholds from cycle7_batch_optimized.yaml
+extends: ../cycle7_batch_optimized.yaml
+
+batch:
+  location: "kandang-atas"
+  root_dir: "../VIDEOS/cycle7/kandang-atas"

dashboard.py

@@ -25,10 +25,15 @@ DEFAULT_SHM_DIR = "/dev/shm"
 DEFAULT_PORT = 8080
 TEMPLATE_DIR = Path(__file__).resolve().parent / "templates"
 
-_db_conn = None
+_thread_local = threading.local()
 _db_lock = threading.Lock()
 _db_path = ""
 _mortality_dirs: list[Path] = []
+_mortality_cache_lock = threading.Lock()
+_mortality_reports_cache: list[Path] = []
+_mortality_reports_mtime: float = 0.0
+_mortality_image_cache: dict[str, Path] = {}
+_MORTALITY_CACHE_TTL: float = 15.0  # seconds
 
 
 def _load_initial_cycle_start_date() -> str:
@@ -88,16 +93,17 @@ def _calc_cycle_info(target_date_str: str) -> dict:
 
 
 def _get_db():
-    global _db_conn
     if not _db_path:
         return None
-    with _db_lock:
-        if _db_conn is None:
-            _db_conn = sqlite3.connect(_db_path, check_same_thread=False)
-            _db_conn.row_factory = sqlite3.Row
-            _db_conn.execute("PRAGMA journal_mode=WAL")
-            _db_conn.execute("PRAGMA cache_size=-8000")
-        return _db_conn
+    conn = getattr(_thread_local, "conn", None)
+    if conn is None:
+        conn = sqlite3.connect(_db_path, timeout=30.0)
+        conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA journal_mode=WAL")
+        conn.execute("PRAGMA busy_timeout=5000")
+        conn.execute("PRAGMA cache_size=-8000")
+        _thread_local.conn = conn
+    return conn
 
 
 def _init_db(db_path: str) -> None:
@@ -406,13 +412,66 @@ class DashboardHandler(SimpleHTTPRequestHandler):
                 data["total_mortality_count"] = sum(r.get("count", 0) for r in data["results"])
             return data
 
-        def _find_report_paths() -> list[Path]:
-            found = []
+        def _find_report_paths(force_refresh: bool = False) -> list[Path]:
+            global _mortality_reports_cache, _mortality_reports_mtime, _mortality_image_cache
+            now = time.time()
+            with _mortality_cache_lock:
+                if not force_refresh and (now - _mortality_reports_mtime) < _MORTALITY_CACHE_TTL and _mortality_reports_cache:
+                    return list(_mortality_reports_cache)
+
+                found = []
+                img_cache: dict[str, Path] = {}
+                for mdir in _mortality_dirs:
+                    p = Path(mdir)
+                    if p.is_dir():
+                        for report in p.rglob("mortality_report.json"):
+                            found.append(report)
+                            # Index images in the same directory as the report
+                            for img in report.parent.glob("output_*.jpg"):
+                                img_cache[img.name] = img
+                            for img in report.parent.glob("output_*.png"):
+                                img_cache[img.name] = img
+
+                _mortality_reports_cache = found
+                _mortality_image_cache = img_cache
+                _mortality_reports_mtime = now
+                return list(found)
+
+        def _find_image_path(filename: str) -> Path | None:
+            # 1. Fast memory cache lookup (O(1))
+            with _mortality_cache_lock:
+                if filename in _mortality_image_cache:
+                    img = _mortality_image_cache[filename]
+                    if img.is_file():
+                        return img
+
+            # 2. Fast check in cached report parent directories
+            reports = _find_report_paths()
+            for r in reports:
+                candidate = r.parent / filename
+                if candidate.is_file():
+                    with _mortality_cache_lock:
+                        _mortality_image_cache[filename] = candidate
+                    return candidate
+
+            # 3. Direct check in base mortality directories
+            for mdir in _mortality_dirs:
+                candidate = Path(mdir) / filename
+                if candidate.is_file():
+                    with _mortality_cache_lock:
+                        _mortality_image_cache[filename] = candidate
+                    return candidate
+
+            # 4. Fallback search and update index
             for mdir in _mortality_dirs:
                 p = Path(mdir)
                 if p.is_dir():
-                    found.extend(list(p.rglob("mortality_report.json")))
-            return found
+                    for img_path in p.rglob(filename):
+                        if img_path.is_file():
+                            with _mortality_cache_lock:
+                                _mortality_image_cache[filename] = img_path
+                            return img_path
+            return None
 
         # --- /api/mortality/history ---
         if path == "/api/mortality/history":
@@ -488,24 +547,21 @@ class DashboardHandler(SimpleHTTPRequestHandler):
             if not (filename.startswith("output_") and filename.lower().endswith((".jpg", ".jpeg", ".png"))):
                 self._send_error(403)
                 return
-            for mdir in _mortality_dirs:
-                p = Path(mdir)
-                if p.is_dir():
-                    for img_path in (p.rglob(filename) if filename else []):
-                        if img_path.is_file():
-                            try:
-                                data = img_path.read_bytes()
-                                ct = mimetypes.guess_type(filename)[0] or "image/jpeg"
-                                self.send_response(200)
-                                self.send_header("Content-Type", ct)
-                                self.send_header("Content-Length", str(len(data)))
-                                self.send_header("Cache-Control", "no-cache")
-                                self.send_header("Access-Control-Allow-Origin", "*")
-                                self.end_headers()
-                                self.wfile.write(data)
-                                return
-                            except (FileNotFoundError, OSError):
-                                pass
+            img_path = _find_image_path(filename)
+            if img_path and img_path.is_file():
+                try:
+                    data = img_path.read_bytes()
+                    ct = mimetypes.guess_type(filename)[0] or "image/jpeg"
+                    self.send_response(200)
+                    self.send_header("Content-Type", ct)
+                    self.send_header("Content-Length", str(len(data)))
+                    self.send_header("Cache-Control", "no-cache")
+                    self.send_header("Access-Control-Allow-Origin", "*")
+                    self.end_headers()
+                    self.wfile.write(data)
+                    return
+                except (FileNotFoundError, OSError):
+                    pass
             self._send_error(404)
             return
 

export_excel_report.py

@@ -17,8 +17,19 @@ def generate_excel_report(db_path, output_excel_path, target_date=None):
     # Remove default sheet
     wb.remove(wb.active)
 
-    # Styles
     font_family = "Segoe UI"
+    if df.empty:
+        ws_empty = wb.create_sheet(title="Daily Summary")
+        ws_empty["A1"] = "Chicken Counter - Summary Report"
+        ws_empty["A1"].font = Font(name=font_family, size=16, bold=True, color="1F4E78")
+        target_info = f" for date {target_date}" if target_date else ""
+        ws_empty["A3"] = f"No batch runs found in database{target_info}."
+        ws_empty["A3"].font = Font(name=font_family, size=11, italic=True)
+        wb.save(output_excel_path)
+        print(f"No records found. Empty report saved to: {output_excel_path}")
+        return
+
+    # Styles
     header_fill = PatternFill(start_color="1F4E78", end_color="1F4E78", fill_type="solid") # Dark Navy
     header_font = Font(name=font_family, size=11, bold=True, color="FFFFFF")
     

run_all_coops.sh

@@ -15,14 +15,16 @@ DATE="${1:-$(date +%Y-%m-%d)}"
 FILTER="${2:-}" # Optional filter like 'K1', 'K2', 'K1-L3', or leave empty for all
 MODE="${3:-parallel_processes}"
 
-# Auto-discover all configured floors in configs/floor_config/ (sorted naturally: K1-L1, K1-L2, K1-L3, etc.)
+# Auto-discover all configured floors in configs/floor_config/ (sorted naturally)
 CONFIGS=()
-for cfg in $(ls -1 "$SCRIPT_DIR/configs/floor_config/"K*-L*.yaml 2>/dev/null | sort -V); do
+for cfg in $(ls -1 "$SCRIPT_DIR/configs/floor_config/"*.yaml 2>/dev/null | sort -V); do
     base_name="$(basename "$cfg" .yaml)"
     if [ -n "$FILTER" ] && [[ "$base_name" != *"$FILTER"* ]]; then
         continue
     fi
     CONFIGS+=("configs/floor_config/$(basename "$cfg")")
+done
+
 # Startup check: Ensure sibling VIDEOS directory exists
 VIDEOS_BASE="$SCRIPT_DIR/../VIDEOS"
 if [ ! -d "$VIDEOS_BASE" ]; then