"""Extract IoT panel rows for Sukawarna cycle 2026-05-22 .. 2026-07-09.""" import json import urllib.request from collections import defaultdict from datetime import datetime, timedelta, timezone BASE = "https://dashboard.cpsp.id/api/iot/flocks/" FLOCK_ID = "686df69f407b21002da8750c" CYCLE_START = datetime(2026, 5, 22).date() CYCLE_END = datetime(2026, 7, 9).date() TZ = timezone(timedelta(hours=7)) def parse_dt(value: str) -> datetime: return datetime.fromisoformat(value.replace("Z", "+00:00")) def fetch_page(page: int) -> dict: with urllib.request.urlopen(f"{BASE}?page={page}", timeout=60) as resp: return json.loads(resp.read()) def to_panel_fields(d: dict) -> dict: """Map API payload.data -> iot_panel columns (no experience_temperature).""" hum = d.get("humidity") or 0 temp = d.get("actualTemperature") or 0 return { "wind_speed": float(d.get("wind") or 0), "humidity": float(hum / 10 if hum > 100 else hum), "water_total": float(d.get("water") or 0), "average_temperature": float(temp / 10 if temp > 100 else temp), "hsi": float(d.get("HSI") or 0), "cycle_day": int(d.get("day") or 0), } def main() -> None: p1 = fetch_page(1) total = p1["count"] page_size = len(p1["results"]) last_page = (total + page_size - 1) // page_size print(f"Scanning {last_page} pages for flock {FLOCK_ID}") print(f"Cycle window: {CYCLE_START} .. {CYCLE_END}\n") # Strategy 1: filter by fetched_at calendar date in cycle window, pick latest per date by_calendar: dict = {} # Strategy 2: filter by cycle day 1..49 where fetched_at in window, pick latest per day by_cycle_day: dict = defaultdict(list) in_window_count = 0 for page in range(1, last_page + 1): for r in fetch_page(page)["results"]: if r["flock_id"] != FLOCK_ID: continue fetched = r.get("fetched_at") if not fetched: continue cal = parse_dt(fetched).astimezone(TZ).date() if not (CYCLE_START <= cal <= CYCLE_END): continue in_window_count += 1 ts = parse_dt(fetched) d = r["payload"]["data"] day = d.get("day") prev = by_calendar.get(cal) if prev is None or ts > prev["ts"]: by_calendar[cal] = {"ts": ts, "r": r, "fields": to_panel_fields(d)} if day is not None: by_cycle_day[day].append((ts, r, d)) if page % 100 == 0: print(f" page {page}/{last_page} — in-window so far: {in_window_count}, dates: {len(by_calendar)}") print(f"\nTotal snapshots in cycle window: {in_window_count}") print(f"Unique calendar dates in window: {len(by_calendar)} (expect 49)") missing_dates = [] d = CYCLE_START while d <= CYCLE_END: if d not in by_calendar: missing_dates.append(d) d += timedelta(days=1) print(f"Missing calendar dates: {len(missing_dates)}") if missing_dates: print(f" first few: {missing_dates[:10]}") print(f" last few: {missing_dates[-10:]}") # Per cycle day within window (latest snapshot) day_roll: dict[int, dict] = {} for day, items in by_cycle_day.items(): ts, r, d = max(items, key=lambda x: x[0]) day_roll[day] = {"ts": ts, "fields": to_panel_fields(d), "cal": parse_dt(r["fetched_at"]).astimezone(TZ).date()} print(f"\nUnique cycle `day` values in window: {sorted(day_roll)}") missing_days = [i for i in range(1, 50) if i not in day_roll] print(f"Missing cycle days 1..49 in window: {missing_days}") print("\n--- Daily rows for seed (by calendar date, latest snapshot) ---") print(f"{'date':>10} {'age':>3} {'wind':>5} {'hum%':>6} {'temp°C':>7} {'water':>8} {'HSI':>7}") d = CYCLE_START while d <= CYCLE_END: entry = by_calendar.get(d) if entry: f = entry["fields"] print( f"{d} {f['cycle_day']:3d} {f['wind_speed']:5.0f} {f['humidity']:6.1f} " f"{f['average_temperature']:7.1f} {f['water_total']:8.0f} {f['hsi']:7.2f}" ) else: age = (d - CYCLE_START).days + 1 print(f"{d} {age:3d} — no API data —") d += timedelta(days=1) out = Path(__file__).resolve().parent / "iot_panel_cycle_extract.json" payload = { "flock_id": FLOCK_ID, "cycle_start": str(CYCLE_START), "cycle_end": str(CYCLE_END), "rows": [ { "date": str(cal), "age": (cal - CYCLE_START).days + 1, **by_calendar[cal]["fields"], } for cal in sorted(by_calendar) ], } out.write_text(json.dumps(payload, indent=2), encoding="utf-8") print(f"\nWrote {len(payload['rows'])} rows to {out}") from pathlib import Path if __name__ == "__main__": main()