feat: clean up duplicate files from dataset, update compiled jsons and recalculate accuracy report

backend/cleanup_duplicates.py

@@ -0,0 +1,86 @@
+import os
+import json
+import subprocess
+
+def main():
+    to_delete = [
+        "1782872141829-rotated_1782872118246_rotated_1782872112296_rotated_1782872107513_CAP3481952560331460805.jpg",
+        "1782909791970-sample_do.jpeg",
+        "IMG_20260701_134646.jpg",
+        "IMG_20260701_135910.jpg",
+        "IMG_20260701_134704.jpg",
+        "IMG_20260701_134704~2.jpg",
+        "IMG_20260701_135950.jpg",
+        "IMG_20260701_134533.jpg",
+        "IMG_20260701_140005.jpg",
+        "IMG_20260701_134817.jpg",
+        "IMG_20260701_135923.jpg",
+        "IMG_20260701_135900.jpg",
+        "IMG_20260701_135938.jpg"
+    ]
+
+    base_dir = "backend"
+    if not os.path.exists(base_dir):
+        base_dir = "."
+
+    uploads_dir = os.path.join(base_dir, "uploads")
+    sources_dir = os.path.join(base_dir, "sources")
+    public_dir = os.path.join(base_dir, "pfm-web-app/public")
+
+    deleted_counts = {
+        "manual_labels_json": 0,
+        "source_images": 0,
+        "public_images": 0
+    }
+
+    # 1. Delete physical files
+    for filename in to_delete:
+        # manual label json
+        ml_path = os.path.join(uploads_dir, f"manual_label_{filename}.json")
+        if os.path.exists(ml_path):
+            os.remove(ml_path)
+            deleted_counts["manual_labels_json"] += 1
+            print(f"Removed manual label: {ml_path}")
+            
+        # source image
+        img_source_path = os.path.join(sources_dir, "test-images", filename)
+        if os.path.exists(img_source_path):
+            os.remove(img_source_path)
+            deleted_counts["source_images"] += 1
+            print(f"Removed source image: {img_source_path}")
+
+        # public image
+        img_public_path = os.path.join(public_dir, "test-images", filename)
+        if os.path.exists(img_public_path):
+            os.remove(img_public_path)
+            deleted_counts["public_images"] += 1
+            print(f"Removed public image: {img_public_path}")
+
+    # 2. Filter test_images_results.jsonl
+    jsonl_path = os.path.join(uploads_dir, "test_images_results.jsonl")
+    if os.path.exists(jsonl_path):
+        remaining_lines = []
+        with open(jsonl_path, "r", encoding="utf-8") as f:
+            for line in f:
+                if not line.strip():
+                    continue
+                try:
+                    data = json.loads(line)
+                    fn = data.get("filename")
+                    if fn not in to_delete:
+                        remaining_lines.append(line)
+                except Exception as e:
+                    print(f"Error parsing line: {e}")
+                    remaining_lines.append(line)
+
+        with open(jsonl_path, "w", encoding="utf-8") as f:
+            f.writelines(remaining_lines)
+        print(f"Filtered results JSONL. Remaining entries: {len(remaining_lines)}")
+
+    print(f"\nCleanup results:")
+    print(f"- Manual label JSON files deleted: {deleted_counts['manual_labels_json']}")
+    print(f"- Source images deleted: {deleted_counts['source_images']}")
+    print(f"- Public served images deleted: {deleted_counts['public_images']}")
+
+if __name__ == "__main__":
+    main()