ai insight rework #2

backend/apps/operations/services/insight_service.py

@@ -109,6 +109,18 @@ ATURAN ANGKA & ARAH (WAJIB DIPATUHI):
 - Panen ≠ kematian; jangan hitung mortalitas dari selisih populasi awal − kini.
 - DILARANG menghitung sendiri, memperkirakan, atau membalik arah tren.
 
+ATURAN STANDAR CP 707 — DILARANG MENGARANG (WAJIB DIPATUHI):
+- Blok STANDAR CP 707 menyatakan PERSIS angka mana yang ada di buku dan mana yang TIDAK.
+- Jika standar menyatakan "TIDAK tercantum di buku" atau "N/A" — DILARANG KERAS mengarang
+  angka standar sendiri (mis. "standar mortalitas 9.32%" atau "standar EEF 235").
+- EEF/IP standar HANYA ada hari ke-7 s/d ke-37 (rentang 327–380). Di luar rentang itu
+  standar TIDAK ADA — jangan mengarang.
+- Mortalitas kumulatif standar hanya sampai hari ke-37 (4.65%). Setelah hari 37 buku hanya
+  memperkirakan +0.1%/hari — ini BUKAN standar resmi. Jika hari >37, tulis "standar tidak
+  tercantum di buku untuk hari ini" dan bandingkan dengan 4.65% + estimasi, bukan mengarang.
+- FCR standar hari >37 gunakan fallback 1.65 (bukan standar buku) — sebutkan "fallback" bukan standar.
+- Bobot standar hanya hari 1–37. Di luar itu "tidak tercantum di buku".
+
 FORMAT OUTPUT (JSON SAJA):
 {"kesimpulan":"...","insight":"..."}
 """
@@ -139,14 +151,151 @@ def build_insight_user_prompt(
     report_type: str,
     report_period: str,
     context: dict[str, Any],
+    graded: dict[str, Any] | None = None,
+    is_end_cycle: bool = False,
+    hypotheses: list[dict[str, Any]] | None = None,
 ) -> str:
-    """User prompt for Ollama — name as-is, never prefix 'kandang' or expose ids."""
+    """Build ordered user prompt template for Ollama.
+
+    Template order (recency = last wins):
+    1. Header: topic, kandang name, period, naming rules
+    2. Data Halaman (JSON) — page data only, no IDs
+    3. STATUS AKTUAL (alert from graded) + FAKTA GRADED (status lines, no unknown when known exists)
+    4. Root Cause / ANALISIS AKAR MASALAH (end_cycle only, from hypotheses)
+    5. Length contract + field role contract
+    6. Output schema (JSON only, placeholders as <...>)
+    """
     # Drop internal ids from the JSON the model sees (defense against id=N narration).
     safe_ctx = {
         k: v
         for k, v in context.items()
         if k not in ("kandangId", "kandang_id", "cycleId", "cycle_id")
     }
+
+    lines: list[str] = []
+    # 1. Header
+    lines.append(
+        f"Buat insight topik `{topic}` untuk unit bernama `{kandang_name}` "
+        f"periode `{report_type}/{report_period}`."
+    )
+    lines.append(
+        f"NAMA WAJIB: tulis PERSIS `{kandang_name}` — jangan menambah kata 'Kandang' di depan "
+        f"(salah: 'Kandang {kandang_name}'; benar: '{kandang_name}'), dan jangan menyebut id."
+    )
+
+    # 2. Data Halaman (JSON)
+    lines.append(f"[Data Halaman (JSON)]\n{json.dumps(safe_ctx, ensure_ascii=False, default=str)}")
+
+    # 3. STATUS AKTUAL + FAKTA GRADED
+    if graded:
+        analyses = graded.get("analyses") or {}
+        known_lines = [
+            f"{name} = {block.get('status')}: {block.get('message')}"
+            for name, block in analyses.items()
+            if isinstance(block, dict)
+            and block.get("message")
+            and block.get("status") != "unknown"
+        ]
+        unknown_lines = [
+            f"{name} = {block.get('status')}: {block.get('message')}"
+            for name, block in analyses.items()
+            if isinstance(block, dict)
+            and block.get("message")
+            and block.get("status") == "unknown"
+        ]
+        status_lines = known_lines or unknown_lines
+        actual_alert = alert_from_graded(graded)
+        lines.append(
+            "\nKONTEKS: semua data adalah ternak AYAM BROILER (unggas) di kandang — "
+            "BUKAN tanaman/pertanian. 'Panen' = panen ayam. 'Bobot' = gram per ekor "
+            "(bukan per karung). 'ADG' = kenaikan bobot harian ayam."
+        )
+        lines.append(
+            f"STATUS AKTUAL: {actual_alert}. Kesimpulan dan insight HARUS konsisten "
+            "dengan status ini: bila warning/critical, sebut masalahnya dan bandingkan "
+            "dengan angka standar CP 707; dilarang menyebut ideal/sehat/baik/aman "
+            "bila STATUS AKTUAL bukan healthy."
+        )
+        if status_lines:
+            lines.append("FAKTA GRADED:\n- " + "\n- ".join(status_lines))
+
+    if topic == "hitung_karung":
+        lines.append(
+            "\nFOKUS KHUSUS HITUNG KARUNG: Narasi dan kesimpulan WAJIB fokus HANYA pada stok/saldo karung pakan, "
+            "karung masuk, saldo awal, dan pakan dituang. JANGAN menyebut, mencari, atau mengeluhkan ketiadaan data mortalitas, "
+            "bobot ayam, atau FCR karena metrik tersebut bukan bagian dari topik Hitung Karung."
+        )
+
+    # 4. Root Cause / ANALISIS AKAR MASALAH (end_cycle only)
+    if is_end_cycle and hypotheses:
+        lines.append("\n[ANALISIS AKAR MASALAH]")
+        for h in hypotheses:
+            hid = h.get("id", "")
+            label = h.get("hipotesis", "")
+            bukti = h.get("bukti", [])
+            conf = h.get("confidence", "")
+            fase = h.get("fase", "")
+            line = f"- {label} ({conf}, fase: {fase})" if conf or fase else f"- {label}"
+            lines.append(line)
+            for b in bukti:
+                lines.append(f"  - {b}")
+
+    # 5. Length contract + field role contract
+    if is_end_cycle:
+        lines.append(
+            "\nPANJANG WAJIB: kesimpulan = 1-3 kalimat PENILAIAN DATA SAJA (tanpa sebab/aksi). "
+            "masalah = bullet isu dari FAKTA GRADED. akar_penyebab = dari ANALISIS AKAR MASALAH saja. "
+            "perbaikan_siklus_berikutnya = dari aksi yang tersedia saja. "
+            "insight = narasi 3-6 kalimat berurutan: "
+            "(1) angka aktual vs standar CP 707, (2) penyebab/implikasi, (3) tindakan konkret. "
+            "Satu kalimat singkat = gagal. DILARANG menulis kata aksi (lakukan, periksa, sebaiknya, "
+            "konsultasikan) di kesimpulan — insight HARUS mengandung minimal satu kata aksi."
+        )
+    else:
+        lines.append(
+            "\nPANJANG WAJIB: insight = narasi 3-6 kalimat berurutan: "
+            "(1) angka aktual vs standar CP 707, (2) penyebab/implikasi, "
+            "(3) tindakan konkret. kesimpulan = 1-3 kalimat PENILAIAN DATA SAJA (tanpa sebab/aksi). "
+            "Satu kalimat singkat = gagal. DILARANG menulis kata aksi (lakukan, periksa, sebaiknya, "
+            "konsultasikan) di kesimpulan — insight HARUS mengandung minimal satu kata aksi."
+        )
+
+    # 6. Output schema
+    if is_end_cycle:
+        lines.append(
+            '\nBALAS HANYA JSON persis: '
+            '{"kesimpulan":"<ringkasan penilaian>",'
+            '"masalah":["<isu dari FAKTA GRADED>"],'
+            '"akar_penyebab":[{"id":"<id dari ANALISIS AKAR MASALAH>","hipotesis":"<label>","bukti":["..."],"confidence":"high|med|low","fase":"brooding|growth|finisher"}],'
+            '"perbaikan_siklus_berikutnya":[{"id":"<id aksi>","fase":"brooding|growth|finisher","aksi":"...","metrik_pantau":"..."}],'
+            '"insight":"<narasi panjang 3-6 kalimat>"}'
+            ' - JANGAN salin teks <> dari contoh; tulis kalimat lengkap sendiri.'
+        )
+    else:
+        lines.append(
+            '\nBALAS HANYA JSON persis: '
+            '{"kesimpulan":"<ringkasan penilaian 1-3 kalimat>",'
+            '"insight":"<narasi panjang 3-6 kalimat: aktual vs standar, penyebab, tindakan>"}'
+            ' - JANGAN salin teks <> dari contoh; tulis kalimat lengkap sendiri.'
+        )
+
+    return "\n".join(lines)
+
+
+def build_insight_user_prompt_legacy(
+    *,
+    topic: str,
+    kandang_name: str,
+    report_type: str,
+    report_period: str,
+    context: dict[str, Any],
+) -> str:
+    """Legacy user prompt (kept for backward compatibility / tests)."""
+    safe_ctx = {
+        k: v
+        for k, v in context.items()
+        if k not in ("kandangId", "kandang_id", "cycleId", "cycle_id")
+    }
     return (
         f"Buat insight topik `{topic}` untuk unit bernama `{kandang_name}` "
         f"periode `{report_type}/{report_period}`.\n"
@@ -217,6 +366,21 @@ def prune_context_by_period(context: dict[str, Any], report_type: str, report_pe
             "Ringkasan KPI per minggu dihitung di backend. Narasikan dari weekly_summaries + "
             "phase_summaries + graded_facts + root-cause candidates; jangan menghitung ulang."
         )
+    else:
+        # Non-end_cycle (page / daily): prune raw history lists to latest 7 days max
+        # so LLM prompt does not bloat into thousands of tokens.
+        for history_key in (
+            "tren_fcr_harian",
+            "tren_eef_harian",
+            "tren_populasi_harian",
+            "tren_harian",
+            "kpiSeries",
+            "tren_kpi_harian",
+            "history",
+        ):
+            val = out.get(history_key)
+            if isinstance(val, list) and len(val) > 7:
+                out[history_key] = val[-7:]
     out["report_type"] = report_type
     out["report_period"] = report_period
     return out
@@ -372,8 +536,8 @@ def _build_phase_summaries(weekly: list[dict[str, Any]]) -> dict[str, Any]:
     return out
 
 
-def grade_context(context: dict[str, Any]) -> dict[str, Any]:
-    analysis = cp707.analyze_with_cp707_standards(context)
+def grade_context(context: dict[str, Any], topic: str | None = None) -> dict[str, Any]:
+    analysis = cp707.analyze_with_cp707_standards(context, topic)
     graded: dict[str, Any] = {
         "kandangName": context.get("kandangName") or context.get("kandang_name"),
         "hari_ke": analysis.get("dayAge") or _day_age(context),
@@ -455,7 +619,8 @@ def call_ollama(system_prompt: str, user_prompt: str) -> tuple[str | None, dict[
         "options": {
             "temperature": float(getattr(settings, "LLM_TEMPERATURE", 0.2) or 0.2),
             "seed": int(getattr(settings, "LLM_SEED", 42) or 42),
-            "num_ctx": int(getattr(settings, "LLM_NUM_CTX", 2048) or 2048),
+            "num_ctx": int(getattr(settings, "LLM_NUM_CTX", 8192) or 8192),
+            "num_predict": int(getattr(settings, "LLM_NUM_PREDICT", 768) or 768),
             "num_thread": int(getattr(settings, "LLM_NUM_THREAD", 4) or 4),
         },
         "messages": [
@@ -753,6 +918,7 @@ _DIAGNOSTIC_KEYWORDS = {
     "bw": "target bobot badan ADG pertumbuhan standar mingguan",
     "iot": "standar ventilasi suhu kelembapan amonia sekam basah kandang",
     "environment": "standar ventilasi suhu kelembapan amonia sekam basah kandang",
+    "feed_balance": "manajemen stok pakan karung pakan harian konsumsi tempat pakan",
 }
 
 
@@ -842,11 +1008,22 @@ def generate_insight(
 
     t_start = time.monotonic()
     _notify(stage_cb, "grading")
-    graded = grade_context(ctx)
+    graded = grade_context(ctx, topic)
     t_after_grade = time.monotonic()
     _notify(stage_cb, "retrieving")
     day = graded.get("hari_ke")
-    standard_block = cp707.build_cp707_standard_block(ctx) or cp707.build_book_reference_context(day or 1)
+    if topic == "hitung_karung":
+        standard_block = (
+            "=== STANDAR MANAJEMEN PAKAN DAN SALDO KARUNG (CP 707) ===\n"
+            f"Umur ayam hari ke-{day or '?'}:\n"
+            "- Stok pakan: Saldo karung pakan di kandang harus selalu positif dan mencukupi kebutuhan harian ayam.\n"
+            "- Pencatatan: Mutasi karung masuk, saldo awal, pakan dituang, dan karung keluar harus tercatat tertib.\n"
+            "- Waspada: Saldo karung menipis (di bawah kebutuhan konsumsi harian) atau saldo negatif menunjukkan perlunya restock atau audit mutasi karung.\n"
+            "- Konsumsi air normal: 2–2.5x dari konsumsi pakan.\n"
+            "=== END STANDAR PAKAN ==="
+        )
+    else:
+        standard_block = cp707.build_cp707_standard_block(ctx) or cp707.build_book_reference_context(day or 1)
 
     is_end_cycle = report_type == "end_cycle"
     hypotheses: list[dict[str, Any]] = []
@@ -862,6 +1039,8 @@ def generate_insight(
             f"panduan CP 707 perbaikan {top_fase or 'broiler'} siklus berikutnya "
             f"umur hari ke-{day or '?'}"
         )
+    elif topic == "hitung_karung":
+        rag_query = f"manajemen stok pakan karung pakan harian konsumsi tempat pakan umur hari ke-{day or '?'}"
     else:
         rag_query = f"panduan manajemen broiler CP 707 untuk {topic} umur hari ke-{day or '?'}"
     rag_query = f"{rag_query} {diagnose_rag_keywords(graded)}".strip()
@@ -904,50 +1083,9 @@ def generate_insight(
         report_type=report_type,
         report_period=report_period,
         context=ctx,
-    )
-    # Page-data JSON ends the user message; small models copy the last JSON they
-    # read. Re-state the required schema AFTER the data so recency wins.
-    # Status/facts go right before it: book chunks (system tail) describe IDEAL
-    # conditions, so without this last the model narrates ideals over actual status.
-    # Skip "unknown" blocks when at least one block has a real status:
-    # no-data lines are padding triggers (e.g. mortality spun into FCR
-    # narrative). All-unknown case keeps them: nothing factual to lean on
-    # and "data tidak tersedia" is then the only correct narration.
-    known_lines = [
-        f"{name} = {block.get('status')}: {block.get('message')}"
-        for name, block in (graded.get("analyses") or {}).items()
-        if isinstance(block, dict)
-        and block.get("message")
-        and block.get("status") != "unknown"
-    ]
-    unknown_lines = [
-        f"{name} = {block.get('status')}: {block.get('message')}"
-        for name, block in (graded.get("analyses") or {}).items()
-        if isinstance(block, dict)
-        and block.get("message")
-        and block.get("status") == "unknown"
-    ]
-    status_lines = known_lines or unknown_lines
-    actual_alert = alert_from_graded(graded)
-    user_prompt = (user_prompt or "").rstrip() + (
-        "\nKONTEKS: semua data adalah ternak AYAM BROILER (unggas) di kandang — "
-        "BUKAN tanaman/pertanian. 'Panen' = panen ayam. 'Bobot' = gram per ekor "
-        "(bukan per karung). 'ADG' = kenaikan bobot harian ayam."
-        f"\nSTATUS AKTUAL: {actual_alert}. Kesimpulan dan insight HARUS konsisten "
-        "dengan status ini: bila warning/critical, sebut masalahnya dan bandingkan "
-        "dengan angka standar CP 707; dilarang menyebut ideal/sehat/baik/aman "
-        "bila STATUS AKTUAL bukan healthy."
-        + ("\nFAKTA GRADED:\n- " + "\n- ".join(status_lines) if status_lines else "")
-        # No length req anywhere -> 3B model sometimes stops after 1 sentence
-        # (three EEF rows stored identical 136/127 chars). Demand structure.
-        + "\nPANJANG WAJIB: insight = narasi 3-6 kalimat berurutan: "
-        "(1) angka aktual vs standar CP 707, (2) penyebab/implikasi, "
-        "(3) tindakan konkret. kesimpulan = 1-3 kalimat. "
-        "Satu kalimat singkat = gagal."
-        # Placeholder <> (not "..."): model has copied schema "..." verbatim
-        # into insight, which stored fine and rendered as an empty page.
-        + '\nBALAS HANYA JSON persis: {"kesimpulan":"<ringkasan>","insight":"<narasi panjang>"}'
-        + ' - JANGAN salin teks <> dari contoh; tulis kalimat lengkap sendiri.'
+        graded=graded,
+        is_end_cycle=is_end_cycle,
+        hypotheses=hypotheses if is_end_cycle else None,
     )
 
     # Re-state output schema as the system prompt's final line (user_prompt