chore(agent): uncommitted changes from task

backend/apps/operations/services/insight_service.py

@@ -104,6 +104,11 @@ ATURAN ANGKA & ARAH (WAJIB DIPATUHI):
 
 FORMAT OUTPUT (JSON SAJA):
 {"kesimpulan":"...","insight":"..."}
+ATURAN FORMAT NARASI (WAJIB — BERLAKU UNTUK SEMUA TOPIK):
+- "kesimpulan" dan "insight" adalah paragraf naratif utuh yang mengalir,
+  sama seperti insight halaman lain — bukan daftar, bukan poin-poin.
+- DILARANG menulis baris per-metrik seperti "fcr: ..." / "bw: ..." dan
+  DILARANG menyalin pesan [GRADED FACTS] mentah — rangkai menjadi analisis utuh.
 """
 
 END_CYCLE_OUTPUT_RULES = """
@@ -462,20 +467,71 @@ def call_ollama(system_prompt: str, user_prompt: str) -> str | None:
         return None
 
 
+def _normalize_string_newlines(fragment: str) -> str:
+    """Replace raw newlines/tabs inside JSON string literals with spaces.
+
+    qwen2.5:3b wraps long string values across real line breaks, which is
+    invalid JSON; without this every long narration fell back to one-liners.
+    """
+    out: list[str] = []
+    in_str = False
+    i = 0
+    while i < len(fragment):
+        ch = fragment[i]
+        if in_str:
+            if ch == "\\" and i + 1 < len(fragment):
+                out.append(ch)
+                out.append(fragment[i + 1])
+                i += 2
+                continue
+            if ch == '"':
+                in_str = False
+                out.append(ch)
+                i += 1
+                continue
+            if ch in "\n\r\t":
+                out.append(" ")
+                i += 1
+                continue
+        elif ch == '"':
+            in_str = True
+        out.append(ch)
+        i += 1
+    return "".join(out)
+
+
 def parse_llm_json(text: str) -> dict[str, str] | None:
     if not text:
         return None
-    cleaned = re.sub(r"<think>[\s\S]*?</think>", "", text, flags=re.I)
-    cleaned = re.sub(r"^```(?:json)?\s*", "", cleaned.strip(), flags=re.I)
+    # Strip fence MARKERS only (FE insightParse parity) — removing the whole
+    # ```…``` block deleted the JSON itself and forced the local fallback.
+    cleaned = re.sub(r"^```(?:json)?\s*", "", text.strip(), flags=re.I)
     cleaned = re.sub(r"\s*```\s*$", "", cleaned)
+    # Tolerate small-model JSON noise (FE parser parity): // and /* */ comments,
+    # trailing commas. Lookbehind keeps "http://" URLs intact.
+    cleaned = re.sub(r"(?<!:)//[^\n]*", "", cleaned)
+    cleaned = re.sub(r"/\*.*?\*/", "", cleaned, flags=re.S)
+    cleaned = re.sub(r",(\s*[}\]])", r"\1", cleaned)
     first = cleaned.find("{")
     last = cleaned.rfind("}")
     if first < 0 or last <= first:
-        return None
+        # No JSON object — the model wrote free prose; keep it as the narrative
+        # instead of dropping it to the per-metric local fallback.
+        prose = cleaned.strip()
+        if len(prose) < 100:
+            return None
+        parts = re.split(r"(?<=[.!?])\s+", prose, maxsplit=1)
+        kesimpulan = parts[0].strip()
+        insight = parts[1].strip() if len(parts) > 1 else kesimpulan
+        return {"kesimpulan": kesimpulan, "insight": insight}
+    fragment = cleaned[first : last + 1]
     try:
-        parsed = json.loads(cleaned[first : last + 1])
+        parsed = json.loads(fragment)
     except json.JSONDecodeError:
-        return None
+        try:
+            parsed = json.loads(_normalize_string_newlines(fragment))
+        except json.JSONDecodeError:
+            return None
     if not isinstance(parsed, dict):
         return None
     kesimpulan = (
@@ -491,7 +547,15 @@ def parse_llm_json(text: str) -> dict[str, str] | None:
         or ""
     )
     if not kesimpulan and not insight:
-        return None
+        # Model invented its own keys (e.g. per-metric fcr/bw/mortality) — fold
+        # the values into narrative paragraphs rather than failing to fallback.
+        values = [str(v).strip() for v in parsed.values() if isinstance(v, str) and v.strip()]
+        if not values:
+            return None
+        return {
+            "kesimpulan": values[0],
+            "insight": "\n\n".join(values[1:]) or values[0],
+        }
     return {
         "kesimpulan": str(kesimpulan) or "Model tidak mengembalikan kesimpulan eksplisit.",
         "insight": str(insight) or "Model tidak mengembalikan rekomendasi eksplisit.",
@@ -608,6 +672,12 @@ def enforce_mortality_wording(
 
 _DUP_KANDANG = re.compile(r"\b[Kk]andang(?:\s+[Kk]andang)+\b")
 
+# Line-start metric labels ("fcr: ...", "bw: ...") — per-metric one-liners the
+# model (or local fallback) may emit instead of narrative paragraphs.
+_METRIC_LABEL_LINE = re.compile(
+    r"(?mi)^\s*(?:fcr|eef|bw|bobot|mortality|mortalitas|environment|ip)\s*:\s*(?=\S)"
+)
+
 
 def collapse_duplicate_kandang(text: str) -> str:
     """Fix LLM slip 'Kandang Kandang 2' → 'Kandang 2' (name already includes Kandang)."""
@@ -616,12 +686,23 @@ def collapse_duplicate_kandang(text: str) -> str:
     return _DUP_KANDANG.sub(lambda m: "Kandang" if m.group(0)[0].isupper() else "kandang", text)
 
 
+def strip_metric_label_lines(text: str) -> str:
+    """Drop `metric: ` line labels and flow the lines into one paragraph."""
+    if not text:
+        return text
+    stripped = _METRIC_LABEL_LINE.sub("", text)
+    if stripped == text:
+        return text
+    return re.sub(r"\s*\n+\s*", " ", stripped).strip()
+
+
 def sanitize_insight_narrative(parsed: dict[str, Any]) -> dict[str, Any]:
     """Deterministic cleanup of narrative fields after the LLM."""
     out = dict(parsed)
     for key in ("kesimpulan", "insight", "summary", "insight_text"):
         if key in out and out[key] is not None:
-            out[key] = collapse_duplicate_kandang(str(out[key]))
+            text = collapse_duplicate_kandang(str(out[key]))
+            out[key] = strip_metric_label_lines(text)
     return out
 
 
@@ -869,6 +950,11 @@ def generate_insight(
     else:
         parsed = parse_llm_json(raw or "")
         if not parsed:
+            logger.warning(
+                "Daily LLM output unparseable (topic=%s); local fallback. raw[:400]=%r",
+                topic,
+                (raw or "")[:400],
+            )
             parsed = local_fallback_insight(graded, topic)
         else:
             parsed = enforce_mortality_wording(parsed, graded)

backend/apps/operations/tests_insight_narrate.py

@@ -0,0 +1,163 @@
+"""Narrate-stage robustness: FCR/EEF daily insights must come out as narrative
+paragraphs, not per-metric one-liners (`fcr: ...`, `bw: ...`)."""
+
+import re
+
+from django.test import SimpleTestCase
+
+from apps.operations.services.insight_service import (
+    ANTI_HALLUCINATION_RULES,
+    local_fallback_insight,
+    parse_llm_json,
+    sanitize_insight_narrative,
+)
+
+_METRIC_ONE_LINER = re.compile(
+    r"(?mi)^\s*(?:fcr|eef|bw|bobot|mortality|mortalitas|environment)\s*:\s*\S"
+)
+
+
+class ParseLlmJsonTests(SimpleTestCase):
+    def test_parses_standard_two_field_json(self):
+        raw = '{"kesimpulan":"Kesimpulan utuh.","insight":"Paragraf insight utuh."}'
+        out = parse_llm_json(raw)
+        self.assertIsNotNone(out)
+        self.assertEqual(out["kesimpulan"], "Kesimpulan utuh.")
+        self.assertEqual(out["insight"], "Paragraf insight utuh.")
+
+    def test_tolerates_trailing_comma_and_line_comment(self):
+        raw = (
+            "{\n"
+            '  "kesimpulan": "Kesimpulan utuh.",\n'
+            '  "insight": "Paragraf insight utuh.",\n'
+            "  // model menulis komentar\n"
+            "}"
+        )
+        out = parse_llm_json(raw)
+        self.assertIsNotNone(out)
+        self.assertEqual(out["kesimpulan"], "Kesimpulan utuh.")
+
+    def test_salvages_metric_keyed_json_into_narrative_without_label_lines(self):
+        """Model echoing graded metrics as keys must not yield one-liners."""
+        raw = (
+            '{"fcr": "FCR 1.70 melebihi standar CP 707 (1.615) pada hari ke-48.",'
+            ' "bw": "Bobot badan aktual 142.1g jauh di bawah standar CP 707 (231g).",'
+            ' "mortality": "Data mortalitas tidak tersedia."}'
+        )
+        out = parse_llm_json(raw)
+        self.assertIsNotNone(out)
+        self.assertTrue(out["kesimpulan"].startswith("FCR 1.70"))
+        self.assertIn("Bobot badan aktual", out["insight"])
+        self.assertIn("Data mortalitas tidak tersedia", out["insight"])
+        self.assertFalse(_METRIC_ONE_LINER.search(out["kesimpulan"]))
+        self.assertFalse(_METRIC_ONE_LINER.search(out["insight"]))
+
+    def test_salvages_prose_response_without_json(self):
+        raw = (
+            "Kandang 1 pada hari ke-8 menunjukkan FCR 0.18 yang lebih baik dari "
+            "standar CP 707 (0.864), sehingga manajemen pakan tampak efisien. "
+            "Namun bobot badan 142.1 gram masih jauh di bawah standar 231 gram "
+            "dan perlu ditindaklanjuti dengan grading segera agar pertumbuhan "
+            "tidak tertinggal dari target siklus."
+        )
+        out = parse_llm_json(raw)
+        self.assertIsNotNone(out)
+        self.assertIn("FCR 0.18", out["kesimpulan"])
+        self.assertIn("grading segera", out["insight"])
+        self.assertFalse(_METRIC_ONE_LINER.search(out["kesimpulan"]))
+
+    def test_tolerates_literal_newlines_inside_json_strings(self):
+        """qwen2.5:3b wraps JSON string values across raw lines — seen live."""
+        raw = (
+            "```json\n"
+            "{\n"
+            '  "kesimpulan": "Pada hari ke-8 di\n'
+            'Kandang 1, FCR 0.18 lebih rendah dari standar CP 707 (0.864).",\n'
+            '  "insight": "Manajemen pakan perlu dijaga\n'
+            'agar FCR tetap ideal hingga panen."\n'
+            "}\n"
+            "```"
+        )
+        out = parse_llm_json(raw)
+        self.assertIsNotNone(out)
+        self.assertIn("Kandang 1", out["kesimpulan"])
+        self.assertIn("FCR 0.18", out["kesimpulan"])
+        self.assertIn("hingga panen", out["insight"])
+
+    def test_short_junk_without_json_returns_none(self):
+        self.assertIsNone(parse_llm_json("Maaf, saya tidak bisa."))
+
+
+class SanitizeNarrativeTests(SimpleTestCase):
+    def test_strips_metric_label_lines_into_paragraph(self):
+        out = sanitize_insight_narrative(
+            {
+                "kesimpulan": (
+                    "bw: Bobot badan aktual 142.1g jauh di bawah standar "
+                    "CP 707 (231g) pada hari ke-8."
+                ),
+                "insight": (
+                    "fcr: FCR 0.18 lebih baik dari standar CP 707 (0.864).\n"
+                    "mortality: Data mortalitas tidak tersedia."
+                ),
+            }
+        )
+        self.assertEqual(
+            out["kesimpulan"],
+            "Bobot badan aktual 142.1g jauh di bawah standar CP 707 (231g) pada hari ke-8.",
+        )
+        self.assertFalse(_METRIC_ONE_LINER.search(out["insight"]))
+        self.assertIn("FCR 0.18", out["insight"])
+        self.assertIn("Data mortalitas tidak tersedia", out["insight"])
+        self.assertNotIn("\n", out["insight"])
+
+    def test_leaves_normal_paragraphs_and_colon_words_alone(self):
+        parsed = {
+            "kesimpulan": "Catatan lapangan: data lengkap.",
+            "insight": "Paragraf pertama.\n\nParagraf kedua.",
+        }
+        out = sanitize_insight_narrative(parsed)
+        self.assertEqual(out["kesimpulan"], parsed["kesimpulan"])
+        self.assertEqual(out["insight"], parsed["insight"])
+
+    def test_still_collapses_duplicate_kandang(self):
+        out = sanitize_insight_narrative(
+            {"kesimpulan": "Kandang Kandang 2 mortalitas tinggi.", "insight": "Pantau."}
+        )
+        self.assertEqual(out["kesimpulan"], "Kandang 2 mortalitas tinggi.")
+
+
+class LocalFallbackNarrativeTests(SimpleTestCase):
+    GRADED = {
+        "analyses": {
+            "bw": {
+                "status": "critical",
+                "message": "Bobot badan aktual 142.1g jauh di bawah standar CP 707 (231g).",
+            },
+            "fcr": {"status": "ok", "message": "FCR 0.18 sesuai standar CP 707 (0.864)."},
+            "mortality": {"status": "unknown", "message": "Data mortalitas tidak tersedia."},
+        }
+    }
+
+    def test_fallback_through_sanitize_has_no_metric_one_liners(self):
+        out = sanitize_insight_narrative(local_fallback_insight(dict(self.GRADED), "fcr"))
+        self.assertFalse(_METRIC_ONE_LINER.search(out["kesimpulan"]))
+        self.assertFalse(_METRIC_ONE_LINER.search(out["insight"]))
+        self.assertIn("Bobot badan aktual", out["kesimpulan"])
+        self.assertIn("FCR 0.18", out["insight"])
+
+    def test_fallback_without_messages_still_reports_unknown(self):
+        out = local_fallback_insight({"analyses": {}}, "eef")
+        self.assertIn("tidak cukup", out["kesimpulan"])
+
+
+class NarrativePromptRuleTests(SimpleTestCase):
+    def test_rules_require_paragraph_output_and_ban_metric_one_liners(self):
+        rules = ANTI_HALLUCINATION_RULES.lower()
+        self.assertIn("paragraf", rules)
+        self.assertIn("fcr: ...", rules)
+
+    def test_end_cycle_format_replace_target_still_present(self):
+        """generate_insight swaps this exact block for end-cycle rules."""
+        target = 'FORMAT OUTPUT (JSON SAJA):\n{"kesimpulan":"...","insight":"..."}'
+        self.assertIn(target, ANTI_HALLUCINATION_RULES)