diff --git a/counter_dashboard.py b/counter_dashboard.py index 104d174..59b75da 100644 --- a/counter_dashboard.py +++ b/counter_dashboard.py @@ -46,6 +46,7 @@ CONTROL_DEFAULT_COUNTING = os.getenv("CONTROL_DEFAULT_COUNTING", "true").lower() SITE_NAME = os.getenv("SITE_NAME", "LIVE") CAMERA_NAME = os.getenv("CAMERA_NAME", "CC1") +OBJECT_LABEL = os.getenv("OBJECT_LABEL", "object") DASHBOARD_PORT = int(os.getenv("DASHBOARD_PORT", "5000")) DASHBOARD_HOST = os.getenv("DASHBOARD_HOST", "0.0.0.0") @@ -429,8 +430,62 @@ def _zone_counts_for_days(conn, dates): return out +def _zero_day_row(date_str, camera_name=None, object_label=None): + """Stub row for a counting day with no activity.""" + counts = _empty_counts() + return { + "date": date_str, + "camera_name": camera_name or CAMERA_NAME, + "object_label": object_label or OBJECT_LABEL, + "total_count": 0, + "counts": counts, + "zones": ZONES_META, + "diff": 0, + "start_time": None, + "end_time": None, + "updated_at": None, + } + + +def _fill_date_gaps(rows, date_from=None, date_to=None, descending=True): + """Ensure every date in [date_from, date_to] is present; missing days are 0.""" + by_date = {r["date"]: r for r in rows} + if date_to is None: + date_to = get_counting_date() + if date_from is None: + if not by_date: + return rows + date_from = min(by_date.keys()) + + sample = next(iter(by_date.values()), None) + cam = sample["camera_name"] if sample else CAMERA_NAME + lab = sample["object_label"] if sample else OBJECT_LABEL + + filled = [] + d = datetime.strptime(date_from, "%Y-%m-%d").date() + end = datetime.strptime(date_to, "%Y-%m-%d").date() + if d > end: + return rows + + while d <= end: + key = d.isoformat() + if key in by_date: + filled.append(by_date[key]) + else: + filled.append(_zero_day_row(key, cam, lab)) + d += timedelta(days=1) + + if descending: + filled.reverse() + return filled + + def _query_history(date_from=None, date_to=None, days=None, limit=None, offset=0): - """Query daily_counters with optional date range and pagination.""" + """Query daily_counters with optional date range and pagination. + + Missing dates in the requested range are filled with zero totals so the API + always returns a continuous series. + """ conn = get_db() cur = conn.cursor() @@ -449,22 +504,14 @@ def _query_history(date_from=None, date_to=None, days=None, limit=None, offset=0 where = f"WHERE {' AND '.join(clauses)}" if clauses else "" - cur.execute(f"SELECT COUNT(*) AS total FROM daily_counters {where}", params) - total = cur.fetchone()["total"] - sql = f""" SELECT counting_date, camera_name, object_label, total_count, start_time, end_time, updated_at FROM daily_counters {where} - ORDER BY counting_date DESC + ORDER BY counting_date ASC """ - page_params = list(params) - if limit is not None: - sql += " LIMIT ? OFFSET ?" - page_params.extend([limit, offset]) - - cur.execute(sql, page_params) + cur.execute(sql, params) day_rows = cur.fetchall() dates = [row["counting_date"] for row in day_rows] zone_map = _zone_counts_for_days(conn, dates) @@ -484,6 +531,12 @@ def _query_history(date_from=None, date_to=None, days=None, limit=None, offset=0 for row in day_rows ] conn.close() + + fill_to = date_to or get_counting_date() + rows = _fill_date_gaps(rows, date_from=date_from, date_to=fill_to, descending=True) + total = len(rows) + if limit is not None: + rows = rows[offset: offset + limit] return rows, total @@ -646,18 +699,11 @@ def api_history_day(counting_date): try: rows, _ = _query_history(date_from=counting_date, date_to=counting_date) - if not rows: - return jsonify({ - "success": False, - "error": { - "code": "NOT_FOUND", - "message": f"No history found for counting date {counting_date}", - }, - }), 404 + data = rows[0] if rows else _zero_day_row(counting_date) return jsonify({ "success": True, "site_name": SITE_NAME, - "data": rows[0] if len(rows) == 1 else rows, + "data": data, }) except sqlite3.OperationalError as e: return jsonify({ @@ -778,29 +824,17 @@ def api_daily_data(): try: days = request.args.get("days", 30, type=int) date_from = (datetime.now() - timedelta(days=days)).date().isoformat() - conn = get_db() - cur = conn.cursor() - cur.execute( - """ - SELECT counting_date, total_count - FROM daily_counters - WHERE counting_date >= ? - ORDER BY counting_date ASC - """, - (date_from,), - ) - day_rows = cur.fetchall() - dates = [row["counting_date"] for row in day_rows] - zone_map = _zone_counts_for_days(conn, dates) + date_to = get_counting_date() + rows, _ = _query_history(date_from=date_from, date_to=date_to) + rows = list(reversed(rows)) # ASC for charts daily_data = [ { - "date": row["counting_date"], + "date": row["date"], "total_count": row["total_count"], - "counts": zone_map.get(row["counting_date"], _empty_counts()), + "counts": row["counts"], } - for row in day_rows + for row in rows ] - conn.close() return jsonify({"zones": ZONES_META, "data": daily_data}) except sqlite3.OperationalError: return jsonify({"zones": ZONES_META, "data": []}), 200 @@ -811,29 +845,17 @@ def api_daily_data(): @app.route("/api/available-dates") def api_available_dates(): try: - conn = get_db() - cur = conn.cursor() - cur.execute( - """ - SELECT counting_date, total_count, start_time, end_time - FROM daily_counters - ORDER BY counting_date DESC - """ - ) - day_rows = cur.fetchall() - dates_list = [row["counting_date"] for row in day_rows] - zone_map = _zone_counts_for_days(conn, dates_list) + rows, _ = _query_history() dates = [ { - "date": row["counting_date"], + "date": row["date"], "total_count": row["total_count"], - "counts": zone_map.get(row["counting_date"], _empty_counts()), + "counts": row["counts"], "start_time": row["start_time"], "end_time": row["end_time"], } - for row in day_rows + for row in rows ] - conn.close() return jsonify({"zones": ZONES_META, "data": dates}) except sqlite3.OperationalError: return jsonify({"zones": ZONES_META, "data": []}), 200 @@ -883,21 +905,9 @@ def export_daily_xlsx(): try: days = request.args.get("days", 30, type=int) date_from = (datetime.now() - timedelta(days=days)).date().isoformat() - conn = get_db() - cur = conn.cursor() - cur.execute( - """ - SELECT counting_date, total_count, start_time, end_time - FROM daily_counters - WHERE counting_date >= ? - ORDER BY counting_date ASC - """, - (date_from,), - ) - rows = cur.fetchall() - dates = [row["counting_date"] for row in rows] - zone_map = _zone_counts_for_days(conn, dates) - conn.close() + date_to = get_counting_date() + rows, _ = _query_history(date_from=date_from, date_to=date_to) + rows = list(reversed(rows)) # ASC for export except sqlite3.OperationalError as e: return jsonify({"success": False, "error": f"Database unavailable: {e}"}), 503 except Exception as e: @@ -918,8 +928,8 @@ def export_daily_xlsx(): _style_header(ws, headers) for r_idx, row in enumerate(rows, 2): - counts = zone_map.get(row["counting_date"], _empty_counts()) - ws.cell(row=r_idx, column=1, value=row["counting_date"]) + counts = row["counts"] + ws.cell(row=r_idx, column=1, value=row["date"]) ws.cell(row=r_idx, column=2, value=row["total_count"]) c = 3 for z in ZONES_META: diff --git a/counter_store.py b/counter_store.py index 96fcf68..f06859a 100644 --- a/counter_store.py +++ b/counter_store.py @@ -48,6 +48,12 @@ class CounterStore: self.db = sqlite3.connect(db_path, check_same_thread=False) self._init_db() self.current_state = self._load_state() + with self.state_lock: + self._fill_missing_days() + counting_date = self.get_counting_date() + if self.current_state is None: + self._start_new_day(counting_date) + self._persist_day() def _table_columns(self, table): cur = self.db.cursor() @@ -246,6 +252,69 @@ class CounterStore: self.save_state() self.log(f'Started counting day {counting_date} ({self.object_label})') + def _ensure_day_row(self, counting_date, start_time=None, end_time=None): + """Insert a zero row for counting_date if it does not already exist.""" + now = datetime.now().isoformat() + cur = self.db.cursor() + cur.execute( + """ + INSERT OR IGNORE INTO daily_counters + (counting_date, camera_name, object_label, + total_count, start_time, end_time) + VALUES (?, ?, ?, 0, ?, ?) + """, + ( + counting_date, + self.camera_name, + self.object_label, + start_time or now, + end_time or now, + ), + ) + for zid in self.zone_ids: + cur.execute( + """ + INSERT OR IGNORE INTO daily_zone_counts + (counting_date, camera_name, object_label, zone_id, count) + VALUES (?, ?, ?, ?, 0) + """, + (counting_date, self.camera_name, self.object_label, zid), + ) + self.db.commit() + + def _fill_missing_days(self): + """Backfill any missing counting dates from first DB row through today as 0.""" + today = self.get_counting_date() + cur = self.db.cursor() + cur.execute( + """ + SELECT counting_date FROM daily_counters + WHERE camera_name = ? AND object_label = ? + ORDER BY counting_date ASC + """, + (self.camera_name, self.object_label), + ) + existing = {row[0] for row in cur.fetchall()} + if not existing: + self._ensure_day_row(today) + return + + start = datetime.strptime(min(existing), '%Y-%m-%d').date() + end = datetime.strptime(today, '%Y-%m-%d').date() + filled = 0 + d = start + while d <= end: + key = d.isoformat() + if key not in existing: + self._ensure_day_row(key) + filled += 1 + d += timedelta(days=1) + if filled: + self.log( + f'Backfilled {filled} zero-activity day(s) ' + f'{start.isoformat()}..{end.isoformat()}' + ) + def record_zone_entry(self, track_id, zone_id): """Record a sack entering a zone. zone_id: 'zone_1' | 'zone_2' | … @@ -323,13 +392,19 @@ class CounterStore: while not self.shutdown_event.is_set(): time.sleep(60) with self.state_lock: + self._fill_missing_days() + counting_date = self.get_counting_date() if self.current_state is None: + self._start_new_day(counting_date) + self._persist_day() continue - if self.current_state['counting_date'] != self.get_counting_date(): + if self.current_state['counting_date'] != counting_date: self.log('Daily cutoff reached - finalizing day totals') self._persist_day() self.current_state = None self.save_state() + self._start_new_day(counting_date) + self._persist_day() def start_cutoff_watcher(self): t = threading.Thread(target=self.cutoff_watcher_loop, daemon=True)