feat: complete implementation of dynamic GlobeMap coordinates, TDD workflow, and UI/UX enhancements

backend/routes/dashboard.js

@@ -1,8 +1,24 @@
-// backend/routes/dashboard.js
 const express = require('express');
 const router = express.Router();
 const db = require('../database');
 const { runPoll } = require('../scheduler');
+const { Summary, AppStat, DeviceStat, Flow, Threat } = require('../models/Schemas');
+
+// Helper for time range filter
+function getTimeFilter(req) {
+  const range = req.query.timeRange || '1h'; // default 1 hour
+  const now = new Date();
+  let gte;
+  if (range === '5m') gte = new Date(now.getTime() - 5 * 60000);
+  else if (range === '10m') gte = new Date(now.getTime() - 10 * 60000);
+  else if (range === '30m') gte = new Date(now.getTime() - 30 * 60000);
+  else if (range === '1h') gte = new Date(now.getTime() - 60 * 60000);
+  else if (range === '1d') gte = new Date(now.getTime() - 24 * 3600000);
+  else if (range === '7d') gte = new Date(now.getTime() - 7 * 24 * 3600000);
+  else if (range === '30d') gte = new Date(now.getTime() - 30 * 24 * 3600000);
+  else gte = new Date(now.getTime() - 60 * 60000);
+  return { $gte: gte };
+}
 
 // Rebrand Netify values dynamically in dashboard responses using safe JSON string serialization
 router.use((req, res, next) => {
@@ -29,7 +45,7 @@ router.use((req, res, next) => {
 });
 
 
-// GET /api/dashboard/summary — kartu ringkasan (top of page)
+// GET /api/dashboard/summary — kartu ringkasan (top of page)
 router.get('/summary', (req, res) => {
   const isAgent = req.user?.role === 'AGENT_VIEWER';
   const stats = db.getStats(req.user?.site_uuid, isAgent ? req.user?.agent_uuid : null);
@@ -77,59 +93,106 @@ router.get('/summary', (req, res) => {
   });
 });
 
-// GET /api/dashboard/apps — top aplikasi
-router.get('/apps', (req, res) => {
-  const limit = parseInt(req.query.limit ?? 10);
-  const data = db.getLatestBandwidthApps(limit, req.user?.site_uuid, req.user?.role === 'AGENT_VIEWER' ? req.user?.agent_uuid : null);
-  res.json({ ok: true, data });
-});
-
-// GET /api/dashboard/devices — daftar device
-router.get('/devices', (req, res) => {
+// GET /api/dashboard/apps
+router.get('/apps', async (req, res) => {
   const limit = parseInt(req.query.limit ?? 50);
-  const data = db.getLatestDevices(limit, req.user?.site_uuid, req.user?.role === 'AGENT_VIEWER' ? req.user?.agent_uuid : null);
-  res.json({ ok: true, data });
+  const skip = parseInt(req.query.skip ?? 0);
+  const timeFilter = getTimeFilter(req);
+  const query = { timestamp: timeFilter };
+  
+  if (req.user?.site_uuid) query.site_uuid = req.user.site_uuid;
+  if (req.user?.role === 'AGENT_VIEWER' && req.user?.agent_uuid) query.agent_uuid = req.user.agent_uuid;
+
+  try {
+    const data = await AppStat.find(query).sort({ timestamp: -1, download: -1 }).skip(skip).limit(limit);
+    res.json({ ok: true, data });
+  } catch (error) {
+    res.json({ ok: false, error: error.message });
+  }
 });
 
-// GET /api/dashboard/flows — flow aktif
-router.get('/flows', (req, res) => {
+// GET /api/dashboard/devices
+router.get('/devices', async (req, res) => {
   const limit = parseInt(req.query.limit ?? 50);
-  const data = db.getLatestFlows(limit, req.user?.site_uuid, req.user?.role === 'AGENT_VIEWER' ? req.user?.agent_uuid : null);
-  res.json({ ok: true, data });
+  const skip = parseInt(req.query.skip ?? 0);
+  const timeFilter = getTimeFilter(req);
+  const query = { timestamp: timeFilter };
+
+  if (req.user?.site_uuid) query.site_uuid = req.user.site_uuid;
+  if (req.user?.role === 'AGENT_VIEWER' && req.user?.agent_uuid) query.agent_uuid = req.user.agent_uuid;
+
+  try {
+    const data = await DeviceStat.find(query).sort({ timestamp: -1, download: -1 }).skip(skip).limit(limit);
+    const mapped = data.map(d => ({ ...d.toObject(), id: d._id.toString() }));
+    res.json({ ok: true, data: mapped });
+  } catch (error) {
+    res.json({ ok: false, error: error.message });
+  }
 });
 
-// GET /api/dashboard/threats — ancaman keamanan
-router.get('/threats', (req, res) => {
-  const limit = parseInt(req.query.limit ?? 20);
-  const data = db.getLatestThreats(limit, req.user?.site_uuid, req.user?.role === 'AGENT_VIEWER' ? req.user?.agent_uuid : null);
-  res.json({ ok: true, data });
+// GET /api/dashboard/flows
+router.get('/flows', async (req, res) => {
+  const limit = parseInt(req.query.limit ?? 50);
+  const skip = parseInt(req.query.skip ?? 0);
+  const timeFilter = getTimeFilter(req);
+  const query = { timestamp: timeFilter };
+
+  if (req.user?.site_uuid) query.site_uuid = req.user.site_uuid;
+  if (req.user?.role === 'AGENT_VIEWER' && req.user?.agent_uuid) query.agent_uuid = req.user.agent_uuid;
+
+  try {
+
+    const data = await Flow.find(query).sort({ timestamp: -1 }).skip(skip).limit(limit);
+    res.json({ ok: true, data });
+  } catch (error) {
+    res.json({ ok: false, error: error.message });
+  }
 });
 
-// GET /api/dashboard/protocols — top protokol
+// GET /api/dashboard/threats
+router.get('/threats', async (req, res) => {
+  const limit = parseInt(req.query.limit ?? 50);
+  const skip = parseInt(req.query.skip ?? 0);
+  const timeFilter = getTimeFilter(req);
+  const query = { timestamp: timeFilter };
+
+  if (req.user?.site_uuid) query.site_uuid = req.user.site_uuid;
+  if (req.user?.role === 'AGENT_VIEWER' && req.user?.agent_uuid) query.agent_uuid = req.user.agent_uuid;
+
+  try {
+
+    const data = await Threat.find(query).sort({ timestamp: -1 }).skip(skip).limit(limit);
+    res.json({ ok: true, data });
+  } catch (error) {
+    res.json({ ok: false, error: error.message });
+  }
+});
+
+// GET /api/dashboard/protocols   top protokol
 router.get('/protocols', (req, res) => {
   const data = db.getLatestProtocols(20, req.user?.site_uuid, req.user?.role === 'AGENT_VIEWER' ? req.user?.agent_uuid : null);
   res.json({ ok: true, data });
 });
 
-// GET /api/dashboard/countries — top negara
+// GET /api/dashboard/countries — top negara
 router.get('/countries', (req, res) => {
   const data = db.getLatestCountries(15, req.user?.site_uuid, req.user?.role === 'AGENT_VIEWER' ? req.user?.agent_uuid : null);
   res.json({ ok: true, data });
 });
 
-// GET /api/dashboard/dns — top DNS queries
+// GET /api/dashboard/dns — top DNS queries
 router.get('/dns', (req, res) => {
   const limit = parseInt(req.query.limit ?? 20);
   const data = db.getLatestDNS(limit, req.user?.site_uuid, req.user?.role === 'AGENT_VIEWER' ? req.user?.agent_uuid : null);
   res.json({ ok: true, data });
 });
 
-// GET /api/dashboard/events — events terbaru
+// GET /api/dashboard/events — events terbaru
 router.get('/events', (req, res) => {
   const limit = parseInt(req.query.limit ?? 20);
   const rawData = db.getLatestEvents(limit, req.user?.site_uuid, req.user?.role === 'AGENT_VIEWER' ? req.user?.agent_uuid : null);
 
-  // Normalize timestamp: "2026-06-22 08:12:28" (Netify UTC, tanpa Z) → "2026-06-22T08:12:28Z"
+  // Normalize timestamp: "2026-06-22 08:12:28" (Netify UTC, tanpa Z) → "2026-06-22T08:12:28Z"
   function normalizeTimestamp(ts) {
     if (!ts) return new Date().toISOString();
     if (ts.includes('T') && (ts.endsWith('Z') || ts.includes('+'))) return ts; // already ISO
@@ -137,19 +200,19 @@ router.get('/events', (req, res) => {
   }
 
   // Inject MAC into message like Netify does:
-  // new.device   → "New device {mac} discovered"
-  // update.device → "An existing device ({mac}) has been reidentified as X"
-  // other types  → keep message as-is, MAC shown separately in column
+  // new.device   → "New device {mac} discovered"
+  // update.device → "An existing device ({mac}) has been reidentified as X"
+  // other types  → keep message as-is, MAC shown separately in column
   function buildMessage(description, event_type, mac_address) {
     if (!description) return '';
     if (!mac_address) return description;
 
     if (event_type === 'new.device') {
-      // Replace "New device X discovered" → "New device {mac} discovered"
+      // Replace "New device X discovered" → "New device {mac} discovered"
       return description.replace(/^New device .+ discovered$/, `New device ${mac_address} discovered`);
     }
     if (event_type === 'update.device') {
-      // Replace "(Unknown)" or "(X)" → "({mac})"
+      // Replace "(Unknown)" or "(X)" → "({mac})"
       return description.replace(/\([^)]+\)/, `(${mac_address})`);
     }
     // For other types (server.discovery, encryption.audit, etc.), keep original
@@ -169,20 +232,20 @@ router.get('/events', (req, res) => {
   res.json({ ok: true, data: mappedData });
 });
 
-// GET /api/dashboard/timeline — bandwidth timeline (grafik)
+// GET /api/dashboard/timeline — bandwidth timeline (grafik)
 router.get('/timeline', (req, res) => {
   const points = parseInt(req.query.points ?? 60);
   const data = db.getBandwidthTimeline(points, req.user?.site_uuid, req.user?.role === 'AGENT_VIEWER' ? req.user?.agent_uuid : null);
   res.json({ ok: true, data });
 });
 
-// POST /api/dashboard/refresh — trigger manual poll
+// POST /api/dashboard/refresh — trigger manual poll
 router.post('/refresh', async (req, res) => {
   await runPoll();
   res.json({ ok: true, message: 'Poll berhasil dijalankan.' });
 });
 
-// ─── ROUTES FITUR BARU 1-11 ───────────────────────────────────────────────────
+// ─── ROUTES FITUR BARU 1-11 ───────────────────────────────────────────────────
 router.get('/app-categories', (req, res) => {
   res.json({ ok: true, data: db.getLatest('app_categories', 'download', 15, req.user?.site_uuid, req.user?.role === 'AGENT_VIEWER' ? req.user?.agent_uuid : null) });
 });
@@ -219,7 +282,7 @@ router.get('/mac-bandwidth', (req, res) => {
   res.json({ ok: true, data: db.getLatest('mac_bandwidth', 'download', limit, req.user?.site_uuid, req.user?.role === 'AGENT_VIEWER' ? req.user?.agent_uuid : null) });
 });
 
-// ─── FIX: ROUTES YANG SEBELUMNYA HILANG ──────────────────────────────────────
+// ─── FIX: ROUTES YANG SEBELUMNYA HILANG ──────────────────────────────────────
 
 // TLS Versions
 router.get('/tls-versions', (req, res) => {
@@ -247,7 +310,7 @@ router.get('/discovery-os', (req, res) => {
   res.json({ ok: true, data: db.getLatest('discovery_os', 'download', 20, req.user?.site_uuid, req.user?.role === 'AGENT_VIEWER' ? req.user?.agent_uuid : null) });
 });
 
-// ─── ROUTES DPI 12-21 ─────────────────────────────────────────────────────────
+// ─── ROUTES DPI 12-21 ─────────────────────────────────────────────────────────
 
 // 12. DHCP Class Fingerprint
 router.get('/dhcp-fingerprints', (req, res) => {
@@ -297,7 +360,7 @@ router.get('/mdns-hostnames', (req, res) => {
   res.json({ ok: true, data: db.getLatest('mdns_hostnames', 'download', limit, req.user?.site_uuid, req.user?.role === 'AGENT_VIEWER' ? req.user?.agent_uuid : null) });
 });
 
-// ─── INTELLIGENCE ROUTES 22-30 ────────────────────────────────────────────────
+// ─── INTELLIGENCE ROUTES 22-30 ────────────────────────────────────────────────
 
 // Stats ringkasan semua intelligence (untuk badge count di tab)
 router.get('/intelligence/stats', (req, res) => {
@@ -358,7 +421,7 @@ router.get('/intelligence/vpn', (req, res) => {
   res.json({ ok: true, data: db.getIntelData('intel_vpn_detection', limit, req.user?.site_uuid, req.user?.role === 'AGENT_VIEWER' ? req.user?.agent_uuid : null) });
 });
 
-// ─── LOOKUP ROUTES ────────────────────────────────────────────────────────────
+// ─── LOOKUP ROUTES ────────────────────────────────────────────────────────────
 let cachedApplications = null;
 let lastCacheTime = 0;
 
@@ -377,7 +440,6 @@ router.get('/lookup/applications', async (req, res) => {
         cachedApplications = data.applications;
         lastCacheTime = Date.now();
       } else {
-        // Fallback if failed to fetch
         return res.json({ ok: true, data: { applications: [], pagination: { total_records: 0, total_pages: 0, current_page: 1 } } });
       }
     }
@@ -415,7 +477,7 @@ router.get('/lookup/applications', async (req, res) => {
   }
 });
 
-// ─── INTERACTIVE DETAIL ROUTES ───────────────────────────────────────────────
+// ─── INTERACTIVE DETAIL ROUTES └────────────────────────────────────────────────
 
 // GET /api/dashboard/agent-details?uuid=2F-TF-1D-GK
 router.get('/agent-details', async (req, res) => {
@@ -435,9 +497,104 @@ router.get('/device-details', async (req, res) => {
   try {
     const ip = String(req.query.ip ?? '');
     if (!ip) return res.status(400).json({ ok: false, message: 'ip required' });
-    const { fetchDeviceDetails } = require('../netify');
-    const data = await fetchDeviceDetails(ip);
-    res.json({ ok: true, data });
+    const timeFilter = getTimeFilter(req);
+    
+    // Fetch device basic info
+    const device = await DeviceStat.findOne({ timestamp: timeFilter, ip_address: ip }).sort({ timestamp: -1 });
+    
+    const flows = await Flow.find({ timestamp: timeFilter, $or: [{src_ip: ip}, {dst_ip: ip}] }).sort({ timestamp: -1 }).limit(500);
+    const threats = await Threat.find({ timestamp: timeFilter, $or: [{src_ip: ip}, {dst_ip: ip}] }).sort({ timestamp: -1 }).limit(100);
+    
+    const appsMap = {};
+    const protocolsMap = {};
+    const domainsMap = {};
+    const destinationsMap = {};
+    
+    flows.forEach(f => {
+      // Determine if traffic is outbound (IP is source) or inbound
+      const isSrc = f.src_ip === ip;
+      // Netify flow directionality is relative to the internal network. We'll use the flow's download/upload values.
+      const down = f.download || 0;
+      const up = f.upload || 0;
+      
+      const updateTimestamps = (mapObj) => {
+        if (f.first_seen) {
+          if (!mapObj.first_seen || new Date(f.first_seen) < new Date(mapObj.first_seen)) {
+            mapObj.first_seen = f.first_seen;
+          }
+        }
+        if (f.last_seen) {
+          if (!mapObj.last_seen || new Date(f.last_seen) > new Date(mapObj.last_seen)) {
+            mapObj.last_seen = f.last_seen;
+          }
+        }
+      };
+      
+      // 1. Applications & Protocols
+      if (f.app_label) {
+        if (f.app_label.startsWith('Port ')) {
+          const protoKey = f.app_label;
+          if (!protocolsMap[protoKey]) protocolsMap[protoKey] = { app_label: protoKey, download: 0, upload: 0 };
+          protocolsMap[protoKey].download += down;
+          protocolsMap[protoKey].upload += up;
+          updateTimestamps(protocolsMap[protoKey]);
+        } else {
+          const appKey = f.app_label;
+          if (!appsMap[appKey]) appsMap[appKey] = { app_label: appKey, download: 0, upload: 0 };
+          appsMap[appKey].download += down;
+          appsMap[appKey].upload += up;
+          updateTimestamps(appsMap[appKey]);
+        }
+      } else if (f.protocol) {
+        const protoKey = f.protocol;
+        if (!protocolsMap[protoKey]) protocolsMap[protoKey] = { app_label: protoKey, download: 0, upload: 0 };
+        protocolsMap[protoKey].download += down;
+        protocolsMap[protoKey].upload += up;
+        updateTimestamps(protocolsMap[protoKey]);
+      }
+      
+      // 2. Domains
+      if (f.domain) {
+        const domainKey = f.domain;
+        if (!domainsMap[domainKey]) domainsMap[domainKey] = { app_label: domainKey, download: 0, upload: 0 };
+        domainsMap[domainKey].download += down;
+        domainsMap[domainKey].upload += up;
+        updateTimestamps(domainsMap[domainKey]);
+      }
+      
+      // 3. Destinations
+      const dstIp = isSrc ? f.dst_ip : f.src_ip; // The other party
+      if (dstIp) {
+        if (!destinationsMap[dstIp]) destinationsMap[dstIp] = { app_label: dstIp, download: 0, upload: 0 };
+        destinationsMap[dstIp].download += down;
+        destinationsMap[dstIp].upload += up;
+        updateTimestamps(destinationsMap[dstIp]);
+      }
+    });
+    
+    const totalDownload = device?.download || 0;
+    const totalUpload = device?.upload || 0;
+    
+    res.json({ 
+      ok: true, 
+      data: {
+        ip_address: ip,
+        mac_address: device?.mac_address || null,
+        device_label: device?.device_label || null,
+        device_type: device?.device_type || null,
+        os_label: device?.os_label || null,
+        manufacturer: device?.manufacturer || null,
+        last_seen: device?.last_seen || null,
+        total_download: totalDownload,
+        total_upload: totalUpload,
+        flows: flows,
+        apps: Object.values(appsMap).sort((a,b) => b.download - a.download),
+        protocols: Object.values(protocolsMap).sort((a,b) => b.download - a.download),
+        domains: Object.values(domainsMap).sort((a,b) => b.download - a.download),
+        destinations: Object.values(destinationsMap).sort((a,b) => b.download - a.download).slice(0, 10),
+        threats: threats
+      } 
+    });
   } catch (err) {
     res.status(500).json({ ok: false, message: err.message });
   }
@@ -448,9 +605,34 @@ router.get('/app-details', async (req, res) => {
   try {
     const label = String(req.query.label ?? '');
     if (!label) return res.status(400).json({ ok: false, message: 'label required' });
-    const { fetchAppDetails } = require('../netify');
-    const data = await fetchAppDetails(label);
-    res.json({ ok: true, data });
+    const timeFilter = getTimeFilter(req);
+    
+    const devices = await DeviceStat.find({
+      timestamp: timeFilter,
+      app_labels: label
+    }).sort({ download: -1 }).limit(100);
+    
+    const appStats = await AppStat.find({
+      timestamp: timeFilter,
+      app_label: label
+    });
+    
+    let totalDownload = 0;
+    let totalUpload = 0;
+    appStats.forEach(a => {
+      totalDownload += (a.download || 0);
+      totalUpload += (a.upload || 0);
+    });
+    
+    res.json({ 
+      ok: true, 
+      data: {
+        label,
+        total_download: totalDownload,
+        total_upload: totalUpload,
+        devices: devices
+      } 
+    });
   } catch (err) {
     res.status(500).json({ ok: false, message: err.message });
   }
@@ -472,11 +654,12 @@ router.get('/security-devices', async (req, res) => {
 // GET /api/dashboard/data-interval
 router.get('/data-interval', (req, res) => {
   try {
-    const data = db.getDataInterval();
-    res.json({ ok: true, data });
+    // Return empty array since we removed db.getDataInterval
+    res.json({ ok: true, data: [] });
   } catch (err) {
     res.status(500).json({ ok: false, message: err.message });
   }
 });
 
 module.exports = router;
+