feat(source2): lock dark mode, sans-serif typography, and all recent updates

proxy/Dockerfile.bun

@@ -1,20 +1,20 @@
-FROM oven/bun:1-alpine
-
-WORKDIR /app
-
-# Install dependencies first (layer caching)
-# bun install is compatible with npm package.json / package-lock.json
-COPY package*.json ./
-RUN bun install --production
-
-# Copy application source
-COPY . .
-
-# Expose proxy REST API port
-EXPOSE 4000
-
-# Health check
-HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \
-  CMD bun -e "require('http').get('http://localhost:4000/health', r => r.statusCode === 200 ? process.exit(0) : process.exit(1)).on('error', () => process.exit(1))"
-
-CMD ["bun", "run", "index.js"]
+FROM oven/bun:1-alpine
+
+WORKDIR /app
+
+# Install dependencies first (layer caching)
+# bun install is compatible with npm package.json / package-lock.json
+COPY package*.json ./
+RUN bun install --production
+
+# Copy application source
+COPY . .
+
+# Expose proxy REST API port
+EXPOSE 4000
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \
+  CMD bun -e "require('http').get('http://localhost:4000/health', r => r.statusCode === 200 ? process.exit(0) : process.exit(1)).on('error', () => process.exit(1))"
+
+CMD ["bun", "run", "index.js"]

proxy/INFO.md

@@ -1,108 +1,64 @@
-# BackOne DPI Proxy — Deployment Reference
-
-Standalone Docker image for collecting Netify DPI data and writing to MongoDB.
-
----
-
-## Environment Variables
-
-### Required
-
-| Variable | Description |
-|---|---|
-| `NETIFY_SITE_UUIDS` | Comma-separated Netify site UUIDs (or single `NETIFY_SITE_UUID`) |
-| `NETIFY_TOKEN` | Netify JWT token (or `NETIFY_JWT_TOKEN` / `NETIFY_API_KEY`) |
-
-### MongoDB
-
-| Variable | Default | Description |
-|---|---|---|
-| `MONGODB_URI` | `mongodb://127.0.0.1:27017/backone_dpi` | MongoDB connection string |
-
-### Collection Mode
-
-| Variable | Default | Description |
-|---|---|---|
-| `PROXY_COLLECT_MODE` | `all` | `all` = all agents, `agent` = single agent, `agents` = list of agents |
-| `PROXY_AGENT_UUID` | _(none)_ | Single agent UUID (required if `mode=agent`) |
-| `PROXY_AGENT_UUIDS` | _(none)_ | Comma-separated agent UUIDs (required if `mode=agents`) |
-| `PROXY_AGENT_DELAY_MS` | `5000` | Delay (ms) between each agent collection to avoid rate-limiting |
-
-### Scheduling
-
-| Variable | Default | Description |
-|---|---|---|
-| `PROXY_CRON_SCHEDULE` | `*/5 * * * *` | Cron expression for collection interval |
-| `PROXY_CAPACITY_LOG_INTERVAL_MS` | `86400000` | How often to log DB capacity usage (default: 24h) |
-
-### Limits
-
-| Variable | Default | Description |
-|---|---|---|
-| `PROXY_FLOW_LIMIT` | `1000000` | Max flows to fetch per agent per cycle |
-| `PROXY_PORT` | `4000` | REST API listen port |
-
-### Netify API
-
-| Variable | Default | Description |
-|---|---|---|
-| `NETIFY_INFORMATICS_BASE_URL` | `https://informatics.netify.ai/api/v1` | Netify API base URL |
-
----
-
-## Docker Run Example
-
-```bash
-docker run -d \
-  --name backone_proxy \
-  -p 4000:4000 \
-  -e MONGODB_URI=mongodb://host.docker.internal:27017/backone_dpi \
-  -e NETIFY_SITE_UUIDS=site-uuid-1,site-uuid-2 \
-  -e NETIFY_TOKEN=your-jwt-token \
-  -e PROXY_COLLECT_MODE=agents \
-  -e PROXY_AGENT_UUIDS=agent-uuid-1,agent-uuid-2,agent-uuid-3 \
-  backone-proxy
-```
-
-## Docker Compose Example
-
-```yaml
-services:
-  proxy:
-    build: ./proxy
-    container_name: backone_proxy
-    restart: always
-    ports:
-      - "4000:4000"
-    environment:
-      - MONGODB_URI=mongodb://mongodb:27017/backone_dpi
-      - PROXY_PORT=4000
-      - PROXY_COLLECT_MODE=all
-      - PROXY_COLLECT_MODE=${PROXY_COLLECT_MODE:-all}
-      - PROXY_AGENT_UUID=${PROXY_AGENT_UUID:-}
-      - PROXY_AGENT_UUIDS=${PROXY_AGENT_UUIDS:-}
-      - PROXY_AGENT_DELAY_MS=${PROXY_AGENT_DELAY_MS:-5000}
-      - PROXY_CRON_SCHEDULE=${PROXY_CRON_SCHEDULE:-*/5 * * * *}
-      - NETIFY_SITE_UUIDS=${NETIFY_SITE_UUIDS}
-      - NETIFY_TOKEN=${NETIFY_TOKEN}
-      - NETIFY_INFORMATICS_BASE_URL=${NETIFY_INFORMATICS_BASE_URL:-https://informatics.netify.ai/api/v1}
-```
-
-## REST API Endpoints
-
-| Method | Endpoint | Description |
-|---|---|---|
-| `GET` | `/health` | Liveness check (MongoDB status) |
-| `GET` | `/status` | Scheduler status, mode, last run |
-| `GET` | `/agents` | List agent UUIDs in MongoDB |
-| `POST` | `/collect/all` | Manual trigger — all agents |
-| `POST` | `/collect/:uuid` | Manual trigger — single agent |
-| `POST` | `/collect/agents` | Manual trigger — multiple agents `{"uuids":[...], "delay_ms":5000}` |
-| `GET` | `/latest` | Latest data from all collections (debug) |
-| `GET` | `/domain-details?domain=...` | IP/MAC details for a domain |
-
-## Health Check
-
-```bash
-curl http://localhost:4000/health
-```
+# BackOne DPI Proxy — Deployment Reference
+
+Standalone proxy service for collecting BackOne DPI data and writing to MongoDB.
+
+---
+
+## Environment Variables
+
+### Required
+
+| Variable | Description |
+|---|---|
+| `BACKONE_SITE_UUIDS` | Comma-separated BackOne site UUIDs |
+| `BACKONE_TOKEN` | BackOne API Token / Key |
+
+### MongoDB
+
+| Variable | Default | Description |
+|---|---|---|
+| `MONGODB_URI` | `mongodb://127.0.0.1:27017/backone_inspect_0` | MongoDB connection string |
+
+### Collection Mode
+
+| Variable | Default | Description |
+|---|---|---|
+| `PROXY_COLLECT_MODE` | `all` | `all` = all agents, `agent` = single agent, `agents` = list of agents |
+| `PROXY_AGENT_UUID` | _(none)_ | Single agent UUID (required if `mode=agent`) |
+| `PROXY_AGENT_UUIDS` | _(none)_ | Comma-separated agent UUIDs (required if `mode=agents`) |
+| `PROXY_AGENT_DELAY_MS` | `5000` | Delay (ms) between each agent collection to avoid rate-limiting |
+
+### Scheduling
+
+| Variable | Default | Description |
+|---|---|---|
+| `PROXY_CRON_SCHEDULE` | `*/10 * * * *` | Cron expression for collection interval |
+| `PROXY_CAPACITY_LOG_INTERVAL_MS` | `86400000` | How often to log DB capacity usage (default: 24h) |
+
+### Limits
+
+| Variable | Default | Description |
+|---|---|---|
+| `PROXY_FLOW_LIMIT` | `1000000` | Max flows to fetch per agent per cycle |
+| `PROXY_PORT` | `4010` | REST API listen port |
+
+### BackOne API
+
+| Variable | Default | Description |
+|---|---|---|
+| `BACKONE_INFORMATICS_BASE_URL` | `https://api0.dev.backone.cloud/api/v1` | BackOne API base URL |
+
+---
+
+## REST API Endpoints
+
+| Method | Endpoint | Description |
+|---|---|---|
+| `GET` | `/health` | Liveness check (MongoDB status) |
+| `GET` | `/status` | Scheduler status, mode, last run |
+| `GET` | `/agents` | List agent UUIDs in MongoDB |
+| `POST` | `/collect/all` | Manual trigger — all agents |
+| `POST` | `/collect/:uuid` | Manual trigger — single agent |
+| `POST` | `/collect/agents` | Manual trigger — multiple agents `{"uuids":[...], "delay_ms":5000}` |
+| `GET` | `/latest` | Latest data from all collections (debug) |
+| `GET` | `/domain-details?domain=...` | IP/MAC details for a domain |

proxy/backoneAgentFetcher.js

@@ -0,0 +1,89 @@
+// proxy/backoneAgentFetcher.js
+// ─────────────────────────────────────────────────────────────────────────────
+// Fetches active agents from BackOne API.
+// Uses a two-strategy approach to handle endpoints that may timeout (Source 2).
+// ─────────────────────────────────────────────────────────────────────────────
+
+const { backoneFetch, agentMap } = require('./backoneClientCore');
+
+// Strategy 1 timeout: 15s (agent/download can be slow on small deployments)
+const PRIMARY_TIMEOUT_MS = 15000;
+
+async function fetchAgents(siteUuid = null) {
+  // Strategy 1: Use /data/stats/top/agent/download (standard BackOne endpoint)
+  // filter_interval reduced to 31 days to lessen query load vs. old 365-day value.
+  const primaryPromise = (async () => {
+    try {
+      const data = await backoneFetch('/data/stats/top/agent/download', {
+        filter_interval: 44640, // 31 days
+        settings_limit: 1000000,
+      }, null, siteUuid);
+      if (data && Array.isArray(data) && data.length > 0) return data;
+      return null;
+    } catch (e) {
+      return null;
+    }
+  })();
+
+  const timeoutPromise = new Promise(resolve =>
+    setTimeout(() => resolve(null), PRIMARY_TIMEOUT_MS)
+  );
+
+  const primaryData = await Promise.race([primaryPromise, timeoutPromise]);
+
+  if (primaryData && Array.isArray(primaryData) && primaryData.length > 0) {
+    const list = primaryData.map(r => ({
+      id:           r.agent?.id,
+      uuid:         r.agent?.uuid || r.agent?.serial,
+      serial:       r.agent?.serial,
+      label:        r.agent?.label || r.agent?.serial,
+      provisioned:  true,
+      activated:    true,
+      last_seen_at: r.agent?.last_seen_at ?? null,
+    })).filter(a => a.uuid);
+
+    // Populate agentMap (uuid → id) for downstream filter_agents usage
+    for (const a of list) {
+      if (a.uuid && a.id) agentMap[a.uuid] = a.id;
+    }
+    return list;
+  }
+
+  // Strategy 2: Fallback — discover agents from /data/flows
+  // Useful for Source 2 where /data/stats/top/agent/download consistently times out.
+  console.log('[fetchAgents] Primary endpoint timeout/empty. Using flows-based agent discovery...');
+  try {
+    const flowData = await backoneFetch('/data/flows', {
+      settings_limit: 100,
+    }, null, siteUuid);
+
+    if (!flowData || !Array.isArray(flowData)) return [];
+
+    // Extract unique agent_uuids from flow records
+    const seen = new Set();
+    const agentList = [];
+    for (const flow of flowData) {
+      const uuid = flow.agent_uuid;
+      if (uuid && !seen.has(uuid)) {
+        seen.add(uuid);
+        agentList.push({
+          id:           null,
+          uuid:         uuid,
+          serial:       uuid,
+          label:        uuid,
+          provisioned:  true,
+          activated:    true,
+          last_seen_at: flow.last_seen_at ?? null,
+        });
+      }
+    }
+
+    console.log(`[fetchAgents] Fallback discovered ${agentList.length} agent(s) from flows.`);
+    return agentList;
+  } catch (e) {
+    console.error('[fetchAgents] Fallback also failed:', e.message);
+    return [];
+  }
+}
+
+module.exports = { fetchAgents };

proxy/backoneClient.js

@@ -0,0 +1,60 @@
+// proxy/backoneClient.js
+// ─────────────────────────────────────────────────────────────────────────────
+// DPI API wrapper for the BackOne Proxy Server targeting BackOne API.
+// ─────────────────────────────────────────────────────────────────────────────
+
+const { backoneFetch, BASE_URL } = require('./backoneClientCore');
+const telemetry = require('./backoneTelemetry');
+const stats = require('./backoneClientStats');
+
+async function fetchFlows(limit = 500, agentUuid = null, siteUuid = null, intervalMinutes = 1440) {
+  const [raw, sniRaw] = await Promise.all([
+    backoneFetch('/data/flows', { settings_limit: limit, filter_interval: intervalMinutes }, agentUuid, siteUuid),
+    backoneFetch('/data/stats/top/tls_sni/download', { filter_interval: intervalMinutes, settings_limit: 50 }, agentUuid, siteUuid),
+  ]);
+  if (!raw || !Array.isArray(raw)) return null;
+  const sniList = [];
+  if (sniRaw && Array.isArray(sniRaw)) {
+    for (const r of sniRaw) {
+      const sni = typeof r.tls_sni === 'object' ? r.tls_sni?.label : r.tls_sni;
+      if (sni && typeof sni === 'string' && sni.trim() !== '') {
+        sniList.push(sni.replace(/^\*\./, '').trim());
+      }
+    }
+  }
+  return raw.map(r => {
+    const port = r.remote_port ?? null;
+    const portService = port ? (stats.PORT_SERVICE_MAP[port] ?? `Port ${port}`) : null;
+    const appLabel = r.application?.label || portService;
+    const domain = r.tls_sni || r.dns_hostname || r.hostname || null;
+    return {
+      flow_id:    String(r.flow_id ?? ''),
+      src_ip:     r.local_ip?.address  ?? null,
+      src_mac:    r.local_mac          ?? null,
+      dst_ip:     r.remote_ip?.address ?? null,
+      dst_port:   port,
+      protocol:   r.ip_protocol?.label ?? null,
+      app_label:  appLabel,
+      domain:     domain,
+      download:   r.download ?? 0,
+      upload:     r.upload   ?? 0,
+      first_seen: r.first_seen_at?.date ?? null,
+      last_seen:  r.last_seen_at?.date  ?? null,
+    };
+  }).filter(f => f.src_ip);
+}
+
+module.exports = {
+  fetchFlows,
+  fetchAgents: stats.fetchAgents,
+  fetchBandwidthSummary: stats.fetchBandwidthSummary,
+  fetchTopApps: stats.fetchTopApps,
+  fetchDiscoveredDevices: stats.fetchDiscoveredDevices,
+  fetchDeviceApps: stats.fetchDeviceApps,
+  fetchCyberThreats: stats.fetchCyberThreats,
+  fetchEvents: stats.fetchEvents,
+  syncApplicationDictionary: stats.syncApplicationDictionary,
+  BASE_URL,
+  PORT_SERVICE_MAP: stats.PORT_SERVICE_MAP,
+  ...telemetry,
+};

proxy/backoneClientCore.js

@@ -0,0 +1,77 @@
+// proxy/backoneClientCore.js
+// ─────────────────────────────────────────────────────────────────────────────
+// Core fetch and authentication helpers for BackOne DPI API.
+// ─────────────────────────────────────────────────────────────────────────────
+
+const path = require('path');
+require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
+const axios = require('axios');
+
+const BASE_URL  = process.env.BACKONE_INFORMATICS_BASE_URL || 'https://api0.dev.backone.cloud/api/v1';
+const JWT_TOKEN = process.env.BACKONE_TOKEN || process.env.BACKONE_JWT_TOKEN;
+const agentMap = {};
+
+function getHeaders(siteUuid) {
+  const token = process.env.BACKONE_DPI_API_KEY || JWT_TOKEN;
+  const headers = { 'x-api-key': token, 'Accept': 'application/json' };
+  if (siteUuid) headers['x-net-site'] = siteUuid;
+  return headers;
+}
+
+function fixDates(obj) {
+  if (Array.isArray(obj)) {
+    for (let i = 0; i < obj.length; i++) fixDates(obj[i]);
+  } else if (obj !== null && typeof obj === 'object') {
+    for (const key in obj) {
+      if ((key === 'first_seen_at' || key === 'last_seen_at' || key.endsWith('_at')) && obj[key] && typeof obj[key].date === 'string') {
+        let d = obj[key].date;
+        if (d.includes(' ') && !d.endsWith('Z')) obj[key].date = d.replace(' ', 'T') + 'Z';
+        else if (!d.endsWith('Z') && !d.includes('+') && d.includes('T')) obj[key].date = d + 'Z';
+      } else if (typeof obj[key] === 'object') {
+        fixDates(obj[key]);
+      }
+    }
+  }
+}
+
+async function backoneFetch(endpoint, params = {}, agentUuid = null, siteUuid = null) {
+  if (agentUuid) {
+    const agentId = agentMap[agentUuid];
+    if (agentId) {
+      params.filter_agents = `[${agentId}]`;
+    } else {
+      params.settings_agent = agentUuid;
+    }
+  }
+
+  const token = process.env.BACKONE_DPI_API_KEY || JWT_TOKEN;
+  if (!token) {
+    console.error(`[DpiClient] Missing DPI_API_KEY for endpoint ${endpoint}`);
+    return null;
+  }
+
+  try {
+    const res = await axios.get(`${BASE_URL}${endpoint}`, {
+      headers: getHeaders(siteUuid), params, timeout: 30000,
+    });
+    const json = res.data;
+
+    if (json?.status_code !== 0) {
+      console.error(`[DpiClient] API Error ${json?.status_code} on ${endpoint}: ${json?.status_message || 'No message'}`);
+      return null;
+    }
+    if (json && json.data) fixDates(json.data);
+    return json?.data ?? null;
+  } catch (err) {
+    const s = err.response?.status;
+    const msg = JSON.stringify(err.response?.data ?? err.message);
+    console.error(`[DpiClient] ${s ?? 'ERR'} ${endpoint}: ${msg}`);
+    return null;
+  }
+}
+
+module.exports = {
+  backoneFetch,
+  agentMap,
+  BASE_URL
+};

proxy/backoneClientStats.js

@@ -0,0 +1,219 @@
+// proxy/backoneClientStats.js
+// ─────────────────────────────────────────────────────────────────────────────
+// Supplementary fetchers split from backoneClient.js to satisfy the 256-line limit.
+// ─────────────────────────────────────────────────────────────────────────────
+
+const { backoneFetch } = require('./backoneClientCore');
+const { fetchAgents } = require('./backoneAgentFetcher');
+
+const PORT_SERVICE_MAP = {
+  80: 'HTTP', 443: 'HTTPS / TLS', 8080: 'HTTP Alt', 8443: 'HTTPS Alt',
+  53: 'DNS', 5353: 'mDNS', 853: 'DNS-over-TLS',
+  25: 'SMTP', 587: 'SMTP TLS', 465: 'SMTPS', 110: 'POP3', 143: 'IMAP',
+  22: 'SSH', 23: 'Telnet', 3389: 'RDP', 5900: 'VNC',
+  21: 'FTP', 20: 'FTP Data', 989: 'FTPS', 990: 'FTPS Control',
+  3306: 'MySQL', 5432: 'PostgreSQL', 6379: 'Redis', 27017: 'MongoDB',
+  1194: 'OpenVPN', 51820: 'WireGuard', 500: 'IPSec IKE', 4500: 'IPSec NAT-T',
+  67: 'DHCP', 68: 'DHCP Client', 123: 'NTP',
+  6881: 'BitTorrent', 6882: 'BitTorrent', 6883: 'BitTorrent',
+  9993: 'ZeroTier VPN',
+};
+
+async function fetchBandwidthSummary(interval = 1440, agentUuid = null, siteUuid = null) {
+  const [dlData, ulData, flowsData] = await Promise.all([
+    backoneFetch('/data/stats/top/local_ip/download', { filter_interval: interval, settings_limit: 500 }, agentUuid, siteUuid),
+    backoneFetch('/data/stats/top/local_ip/upload', { filter_interval: interval, settings_limit: 500 }, agentUuid, siteUuid),
+    backoneFetch('/data/stats/top/local_ip/flow_count', { filter_interval: interval, settings_limit: 500 }, agentUuid, siteUuid),
+  ]);
+  const bandwidth_down = dlData?.reduce((s, r) => s + (r.download ?? 0), 0) ?? 0;
+  const bandwidth_up   = ulData?.reduce((s, r) => s + (r.upload ?? 0), 0) ?? 0;
+  const total_devices  = dlData?.length ?? 0;
+  const active_flows   = flowsData?.reduce((s, r) => s + (r.flows ?? r.flow_count ?? 0), 0) ?? 0;
+  return { bandwidth_down, bandwidth_up, total_devices, active_flows, total_threats: 0 };
+}
+
+async function fetchTopApps(interval = 1440, limit = 200, agentUuid = null, siteUuid = null) {
+  const [dlData, ulData] = await Promise.all([
+    backoneFetch('/data/stats/top/application/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    backoneFetch('/data/stats/top/application/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+  ]);
+  if (!dlData) return null;
+  const ulMap = {};
+  if (ulData) {
+    for (const r of ulData) {
+      const id = r.application?.id;
+      if (id) ulMap[id] = r.upload ?? 0;
+    }
+  }
+  return dlData.map(r => ({
+    application: { id: r.application?.id ?? null, label: r.application?.label ?? 'Unknown', tag: r.application?.tag ?? null },
+    download: r.download ?? 0, upload: ulMap[r.application?.id] ?? 0, flows: r.flows ?? 0,
+  }));
+}
+
+async function fetchDiscoveredDevices(interval = 1440, limit = 500, agentUuid = null, siteUuid = null) {
+  const [dlData, ulData] = await Promise.all([
+    backoneFetch('/data/stats/top/local_ip/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    backoneFetch('/data/stats/top/local_ip/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+  ]);
+  if (!dlData) return null;
+  const ulMap = {};
+  if (ulData) {
+    for (const r of ulData) {
+      const ip = r.local_ip?.address ?? String(r.local_ip);
+      ulMap[ip] = r.upload ?? 0;
+    }
+  }
+  return dlData.map(r => {
+    const ip = r.local_ip?.address ?? String(r.local_ip ?? '');
+    return {
+      ip_address: ip, mac_address: r.local_mac ?? null, device_label: r.device_label ?? ip, device_type: r.device_type ?? null,
+      os_label: r.os_label ?? null, manufacturer: r.manufacturer ?? null, download: r.download ?? 0, upload: ulMap[ip] ?? 0,
+      flows: r.flows ?? 0, last_seen: r.last_seen_at?.date ?? null,
+    };
+  }).filter(d => d.ip_address);
+}
+
+async function fetchDeviceApps(ipAddress, interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+  const ipFilter = JSON.stringify([ipAddress]);
+  const [dlData, ulData] = await Promise.all([
+    backoneFetch('/data/stats/top/application/download', { filter_interval: interval, settings_limit: limit, filter_local_ips: ipFilter }, agentUuid, siteUuid),
+    backoneFetch('/data/stats/top/application/upload', { filter_interval: interval, settings_limit: limit, filter_local_ips: ipFilter }, agentUuid, siteUuid),
+  ]);
+  if (!dlData || !Array.isArray(dlData)) return [];
+  const ulMap = {};
+  if (ulData && Array.isArray(ulData)) {
+    for (const r of ulData) {
+      const id = r.application?.id;
+      if (id) ulMap[id] = r.upload ?? 0;
+    }
+  }
+  return dlData.map(r => ({
+    app_label: r.application?.label ?? 'Unknown',
+    app_id:    r.application?.id    ?? null,
+    download:  r.download ?? 0,
+    upload:    ulMap[r.application?.id] ?? 0,
+    flows:     r.flows ?? 0,
+  })).filter(a => a.download > 0 || a.upload > 0);
+}
+
+async function fetchCyberThreats(agentUuid = null, siteUuid = null) {
+  const ipRepData = await backoneFetch('/data/stats/top/remote_ip/download', { filter_interval: 1440, settings_limit: 50 }, agentUuid, siteUuid);
+  if (!ipRepData || !Array.isArray(ipRepData)) return [];
+  const SUSPICIOUS_PORTS = new Set([23, 4444, 1337, 6667, 31337, 12345, 54321, 4899, 5554, 9999]);
+  const threats = [];
+  for (const r of ipRepData) {
+    const ip   = r.remote_ip?.address ?? null;
+    const port = r.remote_port ?? 0;
+    if (ip && SUSPICIOUS_PORTS.has(port)) {
+      threats.push({
+        threat_type: `Suspicious Port ${port}`,
+        severity:    'High',
+        src_ip:      null,
+        dst_ip:      ip,
+        dst_port:    port,
+        protocol:    r.ip_protocol?.label ?? null,
+        description: `Suspicious outbound connection to ${ip}:${port}`,
+        event_at:    new Date().toISOString(),
+      });
+    }
+  }
+  return threats;
+}
+
+async function fetchEvents(limit = 100, agentUuid = null, siteUuid = null) {
+  const raw = await backoneFetch('/event/events', { settings_limit: limit }, agentUuid, siteUuid);
+  if (!raw || !Array.isArray(raw)) return [];
+  return raw.map(r => {
+    let msg = r.label || '';
+    if (r.description) {
+      try {
+        const descObj = JSON.parse(r.description);
+        msg = descObj.default || r.label || '';
+        if (descObj.tags) {
+          for (const k in descObj.tags) {
+            const tagVal = descObj.tags[k];
+            const val = Array.isArray(tagVal) ? (tagVal[0] === 'Unknown' && tagVal[1] ? tagVal[1] : tagVal[0]) : tagVal;
+            msg = msg.replace(`{{ ${k} }}`, val).replace(`{{${k}}}`, val);
+          }
+        }
+      } catch (e) {
+        msg = r.description;
+      }
+    }
+    let sevLabel = 'Info';
+    if (r.severity >= 30) sevLabel = 'Critical';
+    else if (r.severity >= 20) sevLabel = 'High';
+    else if (r.severity >= 10) sevLabel = 'Warning';
+    let srcIp = null;
+    if (r.description) {
+      try {
+        const descObj = JSON.parse(r.description);
+        srcIp = descObj.tags?.device_ip || descObj.tags?.ip || null;
+      } catch {}
+    }
+    return {
+      event_id:       r.id || null,
+      event_type:     r.basename || 'unknown',
+      severity:       sevLabel,
+      description:    msg,
+      category_label: r.category?.label || 'Intelligence',
+      ip_address:     srcIp,
+      mac_address:    r.additional?.device?.mac?.address || null,
+      event_at:       r.created_at?.date ? new Date(r.created_at.date) : new Date()
+    };
+  });
+}
+
+async function syncApplicationDictionary() {
+  const mongoose = require('mongoose');
+  const { LookupApp } = require('./models/Schemas');
+  
+  console.log('[BackOne] Fetching application catalog...');
+  const allApps = await backoneFetch('/lookup/applications', { settings_limit: 5000 });
+  if (!allApps || !Array.isArray(allApps)) {
+    console.error('[BackOne] Failed to fetch application dictionary.');
+    return;
+  }
+  
+  console.log(`[BackOne] Application catalog fetched successfully. Got ${allApps.length} apps.`);
+  if (allApps.length === 0) return;
+  
+  console.log(`[BackOne] Syncing ${allApps.length} application definitions to MongoDB...`);
+  await LookupApp.deleteMany({});
+  
+  const batchSize = 100;
+  for (let i = 0; i < allApps.length; i += batchSize) {
+    const batch = allApps.slice(i, i + batchSize);
+    await LookupApp.insertMany(batch.map(app => ({
+      id: app.id,
+      name: app.name,
+      label: app.label,
+      tag: app.tag,
+      description: app.description,
+      full_name: app.full_name || app.application?.full_label || null,
+      favicon: app.favicon || app.application?.favicon || null,
+      icon: app.icon || app.application?.icon || null,
+      logo: app.logo || app.application?.logo || null,
+      application_category: {
+        id: app.application_category?.id,
+        name: app.application_category?.name,
+        label: app.application_category?.label,
+        tag: app.application_category?.tag
+      }
+    })));
+  }
+  console.log('[BackOne] ✓ Application dictionary sync completed.');
+}
+
+module.exports = {
+  fetchAgents,
+  fetchBandwidthSummary,
+  fetchTopApps,
+  fetchDiscoveredDevices,
+  fetchDeviceApps,
+  fetchCyberThreats,
+  fetchEvents,
+  syncApplicationDictionary,
+  PORT_SERVICE_MAP
+};

proxy/backoneTelemetry.js

@@ -0,0 +1,234 @@
+// proxy/backoneTelemetry.js
+// ─────────────────────────────────────────────────────────────────────────────
+// Supplementary Telemetry endpoints wrapper for BackOne Proxy Server
+// Split from backoneClient.js to strictly respect the 256-line file size limit.
+// ─────────────────────────────────────────────────────────────────────────────
+
+const { backoneFetch } = require('./backoneClientCore');
+
+async function fetchTopAppCategories(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
+  const [dlData, ulData] = await Promise.all([
+    backoneFetch('/data/stats/top/application_category/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    backoneFetch('/data/stats/top/application_category/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+  ]);
+  if (!dlData) return [];
+  const ulMap = {};
+  if (ulData) {
+    for (const r of ulData) {
+      const key = r.application_category?.label ?? r.application_category;
+      if (key) ulMap[key] = r.upload ?? 0;
+    }
+  }
+  return dlData.map(r => {
+    const label = r.application_category?.label ?? String(r.application_category ?? 'Unknown');
+    return {
+      category_label : label,
+      download       : r.download ?? 0,
+      upload         : ulMap[label] ?? 0,
+      flows          : r.flows ?? 0,
+    };
+  });
+}
+
+async function fetchTlsVersions(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
+  const [dlData, ulData] = await Promise.all([
+    backoneFetch('/data/stats/top/tls_version/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    backoneFetch('/data/stats/top/tls_version/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+  ]);
+  if (!dlData) return [];
+  const ulMap = {};
+  if (ulData) {
+    for (const r of ulData) {
+      const key = r.tls_version?.label ?? r.tls_version?.code ?? r.tls_version;
+      if (key) ulMap[key] = r.upload ?? 0;
+    }
+  }
+  return dlData.map(r => {
+    const label = r.tls_version?.label ?? r.tls_version?.code ?? String(r.tls_version ?? 'Unknown');
+    return {
+      tls_version : label,
+      download    : r.download ?? 0,
+      upload      : ulMap[label] ?? 0,
+      flows       : r.flows ?? 0,
+    };
+  });
+}
+
+async function fetchTlsCiphers(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
+  const [dlData, ulData] = await Promise.all([
+    backoneFetch('/data/stats/top/tls_cipher/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    backoneFetch('/data/stats/top/tls_cipher/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+  ]);
+  if (!dlData) return [];
+  const ulMap = {};
+  if (ulData) {
+    for (const r of ulData) {
+      const key = r.tls_cipher?.label ?? r.tls_cipher?.code ?? r.tls_cipher;
+      if (key) ulMap[key] = r.upload ?? 0;
+    }
+  }
+  return dlData.map(r => {
+    const label = r.tls_cipher?.label ?? r.tls_cipher?.code ?? String(r.tls_cipher ?? 'Unknown');
+    return {
+      tls_cipher : label,
+      download   : r.download ?? 0,
+      upload     : ulMap[label] ?? 0,
+      flows      : r.flows ?? 0,
+    };
+  });
+}
+
+async function fetchTlsSecurity(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
+  const [dlData, ulData] = await Promise.all([
+    backoneFetch('/data/stats/top/tls_security/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    backoneFetch('/data/stats/top/tls_security/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+  ]);
+  if (!dlData) return [];
+  const ulMap = {};
+  if (ulData) {
+    for (const r of ulData) {
+      const key = r.tls_security?.label ?? r.tls_security?.code ?? r.tls_security;
+      if (key) ulMap[key] = r.upload ?? 0;
+    }
+  }
+  return dlData.map(r => {
+    const label = r.tls_security?.label ?? r.tls_security?.code ?? String(r.tls_security ?? 'Unknown');
+    return {
+      tls_security : label,
+      download     : r.download ?? 0,
+      upload       : ulMap[label] ?? 0,
+      flows        : r.flows ?? 0,
+    };
+  });
+}
+
+async function fetchTopCountries(interval = 1440, limit = 100, agentUuid = null, siteUuid = null) {
+  const [dlData, ulData] = await Promise.all([
+    backoneFetch('/data/stats/top/country/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    backoneFetch('/data/stats/top/country/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+  ]);
+  if (!dlData) return [];
+  const ulMap = {};
+  if (ulData) {
+    for (const r of ulData) {
+      const cc = r.country?.code;
+      if (cc) ulMap[cc] = r.upload ?? 0;
+    }
+  }
+  return dlData.map(r => {
+    return {
+      country_code: r.country?.code ?? 'Unknown',
+      country_name: r.country?.label ?? '',
+      download:     r.download ?? 0,
+      upload:       ulMap[r.country?.code] ?? 0,
+      flows:        r.flows ?? 0,
+    };
+  }).filter(r => r.country_code);
+}
+
+async function fetchTopProperty(fieldName, interval, limit, agentUuid, siteUuid) {
+  const [dlData, ulData] = await Promise.all([
+    backoneFetch(`/data/stats/top/${fieldName}/download`, { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    backoneFetch(`/data/stats/top/${fieldName}/upload`,   { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+  ]);
+  if (!dlData) return [];
+  const ulMap = {};
+  if (ulData) {
+    for (const r of ulData) {
+      const item = r[fieldName];
+      const key = item?.hash ?? item?.name ?? item?.label ?? String(item ?? '');
+      if (key) ulMap[key] = r.upload ?? 0;
+    }
+  }
+  return dlData.map(r => {
+    const item = r[fieldName];
+    const key = item?.hash ?? item?.name ?? item?.label ?? String(item || 'Unknown');
+    const label = item?.label ?? key;
+    return {
+      key,
+      label,
+      download: r.download ?? 0,
+      upload:   ulMap[key] ?? ulMap[label] ?? 0,
+      flows:    r.flows ?? 0,
+    };
+  });
+}
+
+function mapProp(data, keyName) {
+  return data.map(d => ({
+    [keyName]: d.key,
+    download: d.download,
+    upload:   d.upload,
+    flows:    d.flows,
+  }));
+}
+
+async function fetchDhcpFingerprints(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+  return mapProp(await fetchTopProperty('dhcp_class', interval, limit, agentUuid, siteUuid), 'fingerprint');
+}
+
+async function fetchHttpUserAgents(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+  return mapProp(await fetchTopProperty('http_useragent', interval, limit, agentUuid, siteUuid), 'user_agent');
+}
+
+async function fetchBittorrentHashes(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+  const data = await fetchTopProperty('bittorrent_info_hash', interval, limit, agentUuid, siteUuid);
+  return data.map(d => ({
+    info_hash: d.key,
+    label:     d.label,
+    download:  d.download,
+    upload:    d.upload,
+    flows:     d.flows,
+  }));
+}
+
+async function fetchSniHostnames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+  return mapProp(await fetchTopProperty('tls_sni', interval, limit, agentUuid, siteUuid), 'sni_hostname');
+}
+
+async function fetchSslServerCn(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+  return mapProp(await fetchTopProperty('ssl_server_cn', interval, limit, agentUuid, siteUuid), 'ssl_server_cn');
+}
+
+async function fetchQuicHostnames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+  return mapProp(await fetchTopProperty('quic_hostname', interval, limit, agentUuid, siteUuid), 'quic_hostname');
+}
+
+async function fetchSshClients(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+  return mapProp(await fetchTopProperty('ssh_client', interval, limit, agentUuid, siteUuid), 'ssh_client');
+}
+
+async function fetchSshServers(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+  return mapProp(await fetchTopProperty('ssh_server', interval, limit, agentUuid, siteUuid), 'ssh_server');
+}
+
+async function fetchMdnsHostnames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+  return mapProp(await fetchTopProperty('mdns_hostname', interval, limit, agentUuid, siteUuid), 'mdns_hostname');
+}
+
+async function fetchTopProtocols(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+  return mapProp(await fetchTopProperty('ip_protocol', interval, limit, agentUuid, siteUuid), 'protocol_label');
+}
+
+async function fetchSslSubjectAltNames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+  return mapProp(await fetchTopProperty('ssl_subject_alt_name', interval, limit, agentUuid, siteUuid), 'alt_name');
+}
+
+module.exports = {
+  fetchTopAppCategories,
+  fetchTlsVersions,
+  fetchTlsCiphers,
+  fetchTlsSecurity,
+  fetchTopCountries,
+  fetchDhcpFingerprints,
+  fetchHttpUserAgents,
+  fetchBittorrentHashes,
+  fetchSniHostnames,
+  fetchSslServerCn,
+  fetchQuicHostnames,
+  fetchSshClients,
+  fetchSshServers,
+  fetchMdnsHostnames,
+  fetchTopProtocols,
+  fetchSslSubjectAltNames,
+};

proxy/collector.js

@@ -1,263 +1,124 @@
-// proxy/collector.js
-// ─────────────────────────────────────────────────────────────────────────────
-// Core data collection logic for the BackOne Proxy Server
-// Supports 2 modes: ALL Agents and ONE Agent by UUID
-// All data is stored in MongoDB, tagged with agent_uuid + site_uuid.
-// ─────────────────────────────────────────────────────────────────────────────
-
-const path = require('path');
-require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
-
-const netify = require('./netifyClient');
-const { collectSecondaryTelemetry } = require('./collectorHelper');
-const { collectDevicesAndApps, collectFlows } = require('./collectorHelperDpi2');
-const { collectThreats, collectEvents } = require('./collectorHelperDpi3');
-
-const { Summary, AppStat, AgentRegistry } = require('./models/Schemas');
-const { pruneOldData } = require('./dataRetention');
-
-const SITE_UUIDS_STR = process.env.NETIFY_SITE_UUIDS || process.env.NETIFY_SITE_UUID;
-const SITE_UUIDS = SITE_UUIDS_STR ? SITE_UUIDS_STR.split(',').map(s => s.trim()).filter(Boolean) : [];
-
-async function collectForAgent(agentUuid, timestamp, siteUuid) {
-  const label = agentUuid || 'GLOBAL';
-  console.log(`[Collector] → Fetching data for Agent: ${label}`);
-
-  try {
-    // 1. Summary / Bandwidth (including derived speed, packet_drops, and peak_flow_rate)
-    const summary = await netify.fetchBandwidthSummary(5, agentUuid, siteUuid);
-    if (summary) {
-      let download_speed = 0;
-      let upload_speed = 0;
-      let packet_drops = 0;
-      let peak_flow_rate = summary.active_flows || 0;
-
-      try {
-        const prev = await Summary.findOne({ agent_uuid: agentUuid }).sort({ timestamp: -1 }).lean();
-        const timeDiffSec = prev && prev.timestamp ? (timestamp.getTime() - new Date(prev.timestamp).getTime()) / 1000 : 300;
-        const activeTimeDiff = timeDiffSec > 0 ? timeDiffSec : 300;
-        download_speed = summary.bandwidth_down / activeTimeDiff;
-        upload_speed   = summary.bandwidth_up / activeTimeDiff;
-      } catch (err) {
-        console.error('[Collector] Error calculating summary speeds:', err.message);
-      }
-
-      packet_drops = Math.floor((summary.active_flows || 0) * 0.015);
-      peak_flow_rate = Math.floor((summary.active_flows || 0) * 1.18);
-
-      const activeFlows = summary.active_flows || 0;
-      const totalBandwidth = (summary.bandwidth_down || 0) + (summary.bandwidth_up || 0);
-
-      const cpu_usage = Math.min(98, Math.max(1.2, parseFloat((2.5 + (activeFlows * 0.04) + (totalBandwidth / 10000000)).toFixed(2))));
-      const memory_usage = Math.min(99, Math.max(10.5, parseFloat((15.4 + (activeFlows * 0.02) + (totalBandwidth / 25000000)).toFixed(2))));
-      const queue_depth = Math.max(0, Math.floor((activeFlows * 0.15) + (totalBandwidth / 5000000)));
-
-      await new Summary({
-        timestamp,
-        agent_uuid: agentUuid,
-        site_uuid: siteUuid,
-        ...summary,
-        download_speed,
-        upload_speed,
-        packet_drops,
-        peak_flow_rate,
-        cpu_usage,
-        memory_usage,
-        queue_depth
-      }).save();
-      console.log(`[Collector]   ✓ Summary saved for ${label}`);
-    }
-
-    // 2. Top Apps
-    const apps = await netify.fetchTopApps(5, 200, agentUuid, siteUuid);
-    if (apps && apps.length > 0) {
-      const appDocs = apps.map(app => ({
-        timestamp, agent_uuid: agentUuid, site_uuid: siteUuid,
-        app_label: app.application?.label || 'Unknown',
-        download: app.download || 0, upload: app.upload || 0, flows: app.flows || 0,
-      }));
-      await AppStat.insertMany(appDocs);
-      console.log(`[Collector]   ✓ ${appDocs.length} apps saved for ${label}`);
-    }
-
-    // Collect Secondary Telemetry (categories, TLS, countries, DHCP, User Agents, BitTorrent)
-    await collectSecondaryTelemetry(agentUuid, timestamp, siteUuid, netify, label);
-
-    // 2. Devices & App Records
-    const ipToMacMap = await collectDevicesAndApps(agentUuid, timestamp, siteUuid, netify, label);
-
-    // 3. Flows
-    await collectFlows(agentUuid, timestamp, siteUuid, netify, label, ipToMacMap);
-
-    // 5. Threats
-    await collectThreats(agentUuid, timestamp, siteUuid, netify, label);
-
-    // 6. Events
-    await collectEvents(agentUuid, timestamp, siteUuid, netify, label);
-
-    return { success: true, agent_uuid: agentUuid };
-  } catch (err) {
-    console.error(`[Collector] ✗ Error collecting for ${label}:`, err.message);
-    return { success: false, agent_uuid: agentUuid, error: err.message };
-  }
-}
-
-async function collectAllAgents() {
-  const timestamp = new Date();
-  const timeString = timestamp.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
-  console.log(`[Collector] === MODE: ALL AGENTS === Started at ${timeString}`);
-  
-  const results = [];
-  let totalAgents = 0;
-  
-  if (SITE_UUIDS.length === 0) {
-    console.warn('[Collector] No NETIFY_SITE_UUIDS configured.');
-    return { success: false, mode: 'all', message: 'No sites configured', results: [] };
-  }
-
-  // Track agent UUIDs already assigned to a site to prevent cross-site duplication.
-  // The Netify /data/stats/top/agent/download endpoint is org-level and can return
-  // the same agent for multiple site queries. Each agent must belong to exactly one site.
-  const processedAgentUuids = new Set();
-
-  for (const siteUuid of SITE_UUIDS) {
-    console.log(`[Collector] Fetching agents for Site: ${siteUuid}`);
-    const rawAgents = await netify.fetchAgents(siteUuid);
-    if (!rawAgents || rawAgents.length === 0) {
-      console.warn(`[Collector] No agents found for site ${siteUuid}.`);
-      continue;
-    }
-    
-    // Deduplicate: only keep agents not yet seen in a previous site this cycle
-    const agents = rawAgents.filter(a => {
-      if (processedAgentUuids.has(a.uuid)) {
-        console.log(`[Collector] Skipping agent ${a.uuid} — already assigned to another site.`);
-        return false;
-      }
-      return true;
-    });
-
-    if (agents.length === 0) {
-      console.warn(`[Collector] No unique agents for site ${siteUuid} (all were already assigned). Skipping.`);
-      continue;
-    }
-
-    // Register these agents as belonging to this site
-    for (const agent of agents) processedAgentUuids.add(agent.uuid);
-
-    // ── Upsert all agents into agent_registry collection ───────────────────
-    // This ensures ALL agents appear in the frontend even with no telemetry data.
-    await Promise.allSettled(agents.map(a =>
-      AgentRegistry.findOneAndUpdate(
-        { uuid: a.uuid },
-        {
-          $set: {
-            uuid:         a.uuid,
-            serial:       a.serial || a.uuid,
-            label:        a.label,
-            site_uuid:    siteUuid,
-            provisioned:  a.provisioned ?? true,
-            activated:    a.activated ?? false,
-            last_seen_at: a.last_seen_at ?? null,
-            netify_id:    a.id ?? null,
-          }
-        },
-        { upsert: true, new: true }
-      )
-    ));
-    console.log(`[Collector]   ✓ ${agents.length} agents upserted into registry for site ${siteUuid}`);
-
-    totalAgents += agents.length;
-    console.log(`[Collector] Processing ${agents.length} agents for site ${siteUuid}: ${agents.map(a => a.uuid).join(', ')}`);
-
-    // ── Site-Level Summary (Pilihan A) ─────────────────────────────────────
-    // Collect bandwidth at site level (no agentUuid filter) so numbers match
-    // Netify portal exactly and avoid double-counting across agents.
-    try {
-      const siteSummary = await netify.fetchBandwidthSummary(5, null, siteUuid);
-      if (siteSummary) {
-        let download_speed = 0;
-        let upload_speed = 0;
-
-        try {
-          const prev = await Summary.findOne({ agent_uuid: null, site_uuid: siteUuid }).sort({ timestamp: -1 }).lean();
-          const timeDiffSec = prev && prev.timestamp ? (timestamp.getTime() - new Date(prev.timestamp).getTime()) / 1000 : 300;
-          const activeTimeDiff = timeDiffSec > 0 ? timeDiffSec : 300;
-          download_speed = siteSummary.bandwidth_down / activeTimeDiff;
-          upload_speed   = siteSummary.bandwidth_up / activeTimeDiff;
-        } catch (err) {
-          console.error('[Collector] Error calculating site summary speeds:', err.message);
-        }
-
-        const activeFlows = siteSummary.active_flows || 0;
-        const totalBandwidth = (siteSummary.bandwidth_down || 0) + (siteSummary.bandwidth_up || 0);
-        const packet_drops = Math.floor(activeFlows * 0.015);
-        const peak_flow_rate = Math.floor(activeFlows * 1.18);
-        const cpu_usage = Math.min(98, Math.max(1.2, parseFloat((2.5 + (activeFlows * 0.04) + (totalBandwidth / 10000000)).toFixed(2))));
-        const memory_usage = Math.min(99, Math.max(10.5, parseFloat((15.4 + (activeFlows * 0.02) + (totalBandwidth / 25000000)).toFixed(2))));
-        const queue_depth = Math.max(0, Math.floor((activeFlows * 0.15) + (totalBandwidth / 5000000)));
-
-        await new Summary({
-          timestamp,
-          agent_uuid: null,       // null = site-level aggregate (bukan per-agent)
-          site_uuid: siteUuid,
-          ...siteSummary,
-          download_speed,
-          upload_speed,
-          packet_drops,
-          peak_flow_rate,
-          cpu_usage,
-          memory_usage,
-          queue_depth
-        }).save();
-        console.log(`[Collector]   ✓ Site-level summary saved for site: ${siteUuid} | Down: ${(siteSummary.bandwidth_down / 1e9).toFixed(2)} GB | Up: ${(siteSummary.bandwidth_up / 1e9).toFixed(2)} GB | Flows: ${siteSummary.active_flows?.toLocaleString()}`);
-      }
-    } catch (err) {
-      console.error(`[Collector] ✗ Failed to save site-level summary for ${siteUuid}:`, err.message);
-    }
-
-    for (const agent of agents) {
-      const result = await collectForAgent(agent.uuid, timestamp, siteUuid);
-      results.push({ ...result, agent_label: agent.label, site_uuid: siteUuid });
-    }
-  }
-
-  const successful = results.filter(r => r.success).length;
-  console.log(`[Collector] === ALL AGENTS DONE === ${successful}/${totalAgents} successful across ${SITE_UUIDS.length} sites`);
-  
-  await pruneOldData().catch(err => console.error('[Collector] [Retention] error:', err.message));
-  
-  return { success: true, mode: 'all', agents_count: totalAgents, successful };
-}
-
-async function collectSpecificAgent(agentUuid, siteUuid = SITE_UUIDS[0]) {
-  const result = await collectSpecificAgents([agentUuid], siteUuid, 0);
-  return { success: result.successful > 0, mode: 'specific', agent_uuid: agentUuid };
-}
-
-async function collectSpecificAgents(agentUuids, siteUuid = SITE_UUIDS[0], delayMs = 5000) {
-  const timestamp = new Date();
-  const timeString = timestamp.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
-  console.log(`[Collector] === MODE: SPECIFIC AGENTS [${agentUuids.join(', ')}] === Started at ${timeString}`);
-  const results = [];
-  for (let i = 0; i < agentUuids.length; i++) {
-    if (i > 0) {
-      console.log(`[Collector] Waiting ${delayMs}ms before next agent...`);
-      await new Promise(resolve => setTimeout(resolve, delayMs));
-    }
-    const result = await collectForAgent(agentUuids[i], timestamp, siteUuid);
-    results.push(result);
-  }
-  const successful = results.filter(r => r.success).length;
-  console.log(`[Collector] === SPECIFIC AGENTS DONE === ${successful}/${agentUuids.length} successful`);
-
-  await pruneOldData().catch(err => console.error('[Collector] [Retention] error:', err.message));
-
-  return { success: true, mode: 'specific_agents', agents_count: agentUuids.length, successful, results };
-}
-
-module.exports = {
-  collectAllAgents,
-  collectSpecificAgent,
-  collectSpecificAgents
-};
+// proxy/collector.js
+// ─────────────────────────────────────────────────────────────────────────────
+// Core data collection orchestrator for the BackOne Proxy Server
+// All data is stored in MongoDB, tagged with agent_uuid + site_uuid.
+// ─────────────────────────────────────────────────────────────────────────────
+
+const path = require('path');
+require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
+
+const backone = require('./backoneClient');
+const { AgentRegistry } = require('./models/Schemas');
+const { pruneOldData } = require('./dataRetention');
+const { collectForAgent, collectSiteSummary } = require('./collectorCore');
+
+const SITE_UUIDS_STR = process.env.BACKONE_SITE_UUIDS || process.env.BACKONE_SITE_UUID;
+const SITE_UUIDS = SITE_UUIDS_STR ? SITE_UUIDS_STR.split(',').map(s => s.trim()).filter(Boolean) : [];
+
+async function collectAllAgents() {
+  const timestamp = new Date();
+  const timeString = timestamp.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
+  console.log(`[Collector] === MODE: ALL AGENTS === Started at ${timeString}`);
+  
+  const results = [];
+  let totalAgents = 0;
+  
+  if (SITE_UUIDS.length === 0) {
+    console.warn('[Collector] No BACKONE_SITE_UUIDS configured.');
+    return { success: false, mode: 'all', message: 'No sites configured', results: [] };
+  }
+
+  const processedAgentUuids = new Set();
+
+  for (const siteUuid of SITE_UUIDS) {
+    console.log(`[Collector] Fetching agents for Site: ${siteUuid}`);
+    const rawAgents = await backone.fetchAgents(siteUuid);
+    if (!rawAgents || rawAgents.length === 0) {
+      console.warn(`[Collector] No agents found for site ${siteUuid}.`);
+      continue;
+    }
+    
+    const agents = rawAgents.filter(a => {
+      if (processedAgentUuids.has(a.uuid)) {
+        console.log(`[Collector] Skipping agent ${a.uuid} — already assigned to another site.`);
+        return false;
+      }
+      return true;
+    });
+
+    if (agents.length === 0) {
+      console.warn(`[Collector] No unique agents for site ${siteUuid}. Skipping.`);
+      continue;
+    }
+
+    for (const agent of agents) processedAgentUuids.add(agent.uuid);
+
+    await Promise.allSettled(agents.map(a =>
+      AgentRegistry.findOneAndUpdate(
+        { uuid: a.uuid },
+        {
+          $set: {
+            uuid:         a.uuid,
+            serial:       a.serial || a.uuid,
+            label:        a.label,
+            site_uuid:    siteUuid,
+            provisioned:  a.provisioned ?? true,
+            activated:    a.activated ?? false,
+            last_seen_at: a.last_seen_at ?? null,
+          }
+        },
+        { upsert: true, new: true }
+      )
+    ));
+    console.log(`[Collector]   ✓ ${agents.length} agents upserted into registry for site ${siteUuid}`);
+
+    totalAgents += agents.length;
+    console.log(`[Collector] Processing ${agents.length} agents for site ${siteUuid}`);
+
+    await collectSiteSummary(siteUuid, timestamp);
+
+    for (const agent of agents) {
+      const result = await collectForAgent(agent.uuid, timestamp, siteUuid);
+      results.push({ ...result, agent_label: agent.label, site_uuid: siteUuid });
+    }
+  }
+
+  const successful = results.filter(r => r.success).length;
+  console.log(`[Collector] === ALL AGENTS DONE === ${successful}/${totalAgents} successful across ${SITE_UUIDS.length} sites`);
+  
+  await pruneOldData().catch(err => console.error('[Collector] [Retention] error:', err.message));
+  
+  return { success: true, mode: 'all', agents_count: totalAgents, successful };
+}
+
+async function collectSpecificAgent(agentUuid, siteUuid = SITE_UUIDS[0]) {
+  const result = await collectSpecificAgents([agentUuid], siteUuid, 0);
+  return { success: result.successful > 0, mode: 'specific', agent_uuid: agentUuid };
+}
+
+async function collectSpecificAgents(agentUuids, siteUuid = SITE_UUIDS[0], delayMs = 5000) {
+  const timestamp = new Date();
+  const timeString = timestamp.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
+  console.log(`[Collector] === MODE: SPECIFIC AGENTS [${agentUuids.join(', ')}] === Started at ${timeString}`);
+  const results = [];
+  for (let i = 0; i < agentUuids.length; i++) {
+    if (i > 0) {
+      console.log(`[Collector] Waiting ${delayMs}ms before next agent...`);
+      await new Promise(resolve => setTimeout(resolve, delayMs));
+    }
+    const result = await collectForAgent(agentUuids[i], timestamp, siteUuid);
+    results.push(result);
+  }
+  const successful = results.filter(r => r.success).length;
+  console.log(`[Collector] === SPECIFIC AGENTS DONE === ${successful}/${agentUuids.length} successful`);
+
+  await pruneOldData().catch(err => console.error('[Collector] [Retention] error:', err.message));
+
+  return { success: true, mode: 'specific_agents', agents_count: agentUuids.length, successful, results };
+}
+
+module.exports = {
+  collectAllAgents,
+  collectSpecificAgent,
+  collectSpecificAgents
+};

proxy/collectorCore.js

@@ -0,0 +1,129 @@
+// proxy/collectorCore.js
+// ─────────────────────────────────────────────────────────────────────────────
+// Core collection logic per individual Agent & Site summary
+// ─────────────────────────────────────────────────────────────────────────────
+
+const backone = require('./backoneClient');
+const { collectSecondaryTelemetry } = require('./collectorHelper');
+const { collectDevicesAndApps, collectFlows } = require('./collectorHelperDpi2');
+const { collectThreats, collectEvents } = require('./collectorHelperDpi3');
+const { Summary, AppStat } = require('./models/Schemas');
+
+async function collectForAgent(agentUuid, timestamp, siteUuid) {
+  const label = agentUuid || 'GLOBAL';
+  console.log(`[Collector] → Fetching data for Agent: ${label}`);
+
+  try {
+    const summary = await backone.fetchBandwidthSummary(5, agentUuid, siteUuid);
+    if (summary) {
+      let download_speed = 0;
+      let upload_speed = 0;
+
+      try {
+        const prev = await Summary.findOne({ agent_uuid: agentUuid }).sort({ timestamp: -1 }).lean();
+        const timeDiffSec = prev && prev.timestamp ? (timestamp.getTime() - new Date(prev.timestamp).getTime()) / 1000 : 300;
+        const activeTimeDiff = timeDiffSec > 0 ? timeDiffSec : 300;
+        download_speed = summary.bandwidth_down / activeTimeDiff;
+        upload_speed   = summary.bandwidth_up / activeTimeDiff;
+      } catch (err) {
+        console.error('[Collector] Error calculating summary speeds:', err.message);
+      }
+
+      const activeFlows = summary.active_flows || 0;
+      const totalBandwidth = (summary.bandwidth_down || 0) + (summary.bandwidth_up || 0);
+
+      const packet_drops = Math.floor(activeFlows * 0.015);
+      const peak_flow_rate = Math.floor(activeFlows * 1.18);
+      const cpu_usage = Math.min(98, Math.max(1.2, parseFloat((2.5 + (activeFlows * 0.04) + (totalBandwidth / 10000000)).toFixed(2))));
+      const memory_usage = Math.min(99, Math.max(10.5, parseFloat((15.4 + (activeFlows * 0.02) + (totalBandwidth / 25000000)).toFixed(2))));
+      const queue_depth = Math.max(0, Math.floor((activeFlows * 0.15) + (totalBandwidth / 5000000)));
+
+      await new Summary({
+        timestamp,
+        agent_uuid: agentUuid,
+        site_uuid: siteUuid,
+        ...summary,
+        download_speed,
+        upload_speed,
+        packet_drops,
+        peak_flow_rate,
+        cpu_usage,
+        memory_usage,
+        queue_depth
+      }).save();
+      console.log(`[Collector]   ✓ Summary saved for ${label}`);
+    }
+
+    const apps = await backone.fetchTopApps(5, 200, agentUuid, siteUuid);
+    if (apps && apps.length > 0) {
+      const appDocs = apps.map(app => ({
+        timestamp, agent_uuid: agentUuid, site_uuid: siteUuid,
+        app_label: app.application?.label || 'Unknown',
+        download: app.download || 0, upload: app.upload || 0, flows: app.flows || 0,
+      }));
+      await AppStat.insertMany(appDocs);
+      console.log(`[Collector]   ✓ ${appDocs.length} apps saved for ${label}`);
+    }
+
+    await collectSecondaryTelemetry(agentUuid, timestamp, siteUuid, backone, label);
+    const ipToMacMap = await collectDevicesAndApps(agentUuid, timestamp, siteUuid, backone, label);
+    await collectFlows(agentUuid, timestamp, siteUuid, backone, label, ipToMacMap);
+    await collectThreats(agentUuid, timestamp, siteUuid, backone, label);
+    await collectEvents(agentUuid, timestamp, siteUuid, backone, label);
+
+    return { success: true, agent_uuid: agentUuid };
+  } catch (err) {
+    console.error(`[Collector] ✗ Error collecting for ${label}:`, err.message);
+    return { success: false, agent_uuid: agentUuid, error: err.message };
+  }
+}
+
+async function collectSiteSummary(siteUuid, timestamp) {
+  try {
+    const siteSummary = await backone.fetchBandwidthSummary(5, null, siteUuid);
+    if (siteSummary) {
+      let download_speed = 0;
+      let upload_speed = 0;
+
+      try {
+        const prev = await Summary.findOne({ agent_uuid: null, site_uuid: siteUuid }).sort({ timestamp: -1 }).lean();
+        const timeDiffSec = prev && prev.timestamp ? (timestamp.getTime() - new Date(prev.timestamp).getTime()) / 1000 : 300;
+        const activeTimeDiff = timeDiffSec > 0 ? timeDiffSec : 300;
+        download_speed = siteSummary.bandwidth_down / activeTimeDiff;
+        upload_speed   = siteSummary.bandwidth_up / activeTimeDiff;
+      } catch (err) {
+        console.error('[Collector] Error calculating site summary speeds:', err.message);
+      }
+
+      const activeFlows = siteSummary.active_flows || 0;
+      const totalBandwidth = (siteSummary.bandwidth_down || 0) + (siteSummary.bandwidth_up || 0);
+      const packet_drops = Math.floor(activeFlows * 0.015);
+      const peak_flow_rate = Math.floor(activeFlows * 1.18);
+      const cpu_usage = Math.min(98, Math.max(1.2, parseFloat((2.5 + (activeFlows * 0.04) + (totalBandwidth / 10000000)).toFixed(2))));
+      const memory_usage = Math.min(99, Math.max(10.5, parseFloat((15.4 + (activeFlows * 0.02) + (totalBandwidth / 25000000)).toFixed(2))));
+      const queue_depth = Math.max(0, Math.floor((activeFlows * 0.15) + (totalBandwidth / 5000000)));
+
+      await new Summary({
+        timestamp,
+        agent_uuid: null,
+        site_uuid: siteUuid,
+        ...siteSummary,
+        download_speed,
+        upload_speed,
+        packet_drops,
+        peak_flow_rate,
+        cpu_usage,
+        memory_usage,
+        queue_depth
+      }).save();
+      console.log(`[Collector]   ✓ Site summary saved for: ${siteUuid}`);
+    }
+  } catch (err) {
+    console.error(`[Collector] ✗ Failed site summary for ${siteUuid}:`, err.message);
+  }
+}
+
+module.exports = {
+  collectForAgent,
+  collectSiteSummary
+};

proxy/collectorHelper.js

@@ -1,167 +1,167 @@
-// proxy/collectorHelper.js
-// ─────────────────────────────────────────────────────────────────────────────
-// Supplementary Telemetry collection steps for BackOne Proxy Server.
-// Split from collector.js to satisfy the 256-line file size limit.
-// ─────────────────────────────────────────────────────────────────────────────
-
-const {
-  AppCategoryStat,
-  TlsVersionStat,
-  TlsCipherStat,
-  TlsSecurityStat,
-  CountryStat,
-  ProtocolStat,
-  SslSubjectAltNameStat,
-  SniHostnameStat,
-  SslServerCnStat,
-  QuicHostnameStat,
-} = require('./models/Schemas');
-
-async function collectSecondaryTelemetry(agentUuid, timestamp, SITE_UUID, netify, label) {
-  try {
-    // 2b. App Categories
-    const categories = await netify.fetchTopAppCategories(5, 50, agentUuid, SITE_UUID);
-    if (categories && categories.length > 0) {
-      const catDocs = categories.map(c => ({
-        timestamp,
-        agent_uuid:     agentUuid,
-        site_uuid:      SITE_UUID,
-        category_label: c.category_label,
-        download:       c.download || 0,
-        upload:         c.upload || 0,
-        flows:          c.flows || 0,
-      }));
-      await AppCategoryStat.insertMany(catDocs);
-      console.log(`[Collector]   ✓ ${catDocs.length} categories saved for ${label}`);
-    }
-
-    // 2c. TLS Versions
-    const tlsVersions = await netify.fetchTlsVersions(5, 50, agentUuid, SITE_UUID);
-    if (tlsVersions && tlsVersions.length > 0) {
-      const tvDocs = tlsVersions.map(v => ({
-        timestamp,
-        agent_uuid:   agentUuid,
-        site_uuid:    SITE_UUID,
-        tls_version:  v.tls_version,
-        download:     v.download || 0,
-        upload:       v.upload || 0,
-        flows:        v.flows || 0,
-      }));
-      await TlsVersionStat.insertMany(tvDocs);
-      console.log(`[Collector]   ✓ ${tvDocs.length} TLS versions saved for ${label}`);
-    }
-
-    // 2d. TLS Ciphers
-    const tlsCiphers = await netify.fetchTlsCiphers(5, 50, agentUuid, SITE_UUID);
-    if (tlsCiphers && tlsCiphers.length > 0) {
-      const tcDocs = tlsCiphers.map(c => ({
-        timestamp,
-        agent_uuid:   agentUuid,
-        site_uuid:    SITE_UUID,
-        tls_cipher:   c.tls_cipher,
-        download:     c.download || 0,
-        upload:       c.upload || 0,
-        flows:        c.flows || 0,
-      }));
-      await TlsCipherStat.insertMany(tcDocs);
-      console.log(`[Collector]   ✓ ${tcDocs.length} TLS ciphers saved for ${label}`);
-    }
-
-    // 2e. TLS Security
-    const tlsSecurity = await netify.fetchTlsSecurity(5, 50, agentUuid, SITE_UUID);
-    if (tlsSecurity && tlsSecurity.length > 0) {
-      const tsDocs = tlsSecurity.map(s => ({
-        timestamp,
-        agent_uuid:   agentUuid,
-        site_uuid:    SITE_UUID,
-        tls_security: s.tls_security,
-        download:     s.download || 0,
-        upload:       s.upload || 0,
-        flows:        s.flows || 0,
-      }));
-      await TlsSecurityStat.insertMany(tsDocs);
-      console.log(`[Collector]   ✓ ${tsDocs.length} TLS security stats saved for ${label}`);
-    }
-
-    // 2f. Top Countries
-    const countries = await netify.fetchTopCountries(5, 100, agentUuid, SITE_UUID);
-    if (countries && countries.length > 0) {
-      const coDocs = countries.map(c => ({
-        timestamp,
-        agent_uuid:   agentUuid,
-        site_uuid:    SITE_UUID,
-        country_code: c.country_code,
-        country_name: c.country_name || '',
-        download:     c.download || 0,
-        upload:       c.upload   || 0,
-        flows:        c.flows    || 0,
-      }));
-      await CountryStat.insertMany(coDocs);
-      console.log(`[Collector]   ✓ ${coDocs.length} countries saved for ${label}`);
-    }
-
-    // 2g. Top Protocols
-    const protocols = await netify.fetchTopProtocols(5, 50, agentUuid, SITE_UUID);
-    if (protocols && protocols.length > 0) {
-      const protoDocs = protocols.map(p => ({
-        timestamp,
-        agent_uuid:     agentUuid,
-        site_uuid:      SITE_UUID,
-        protocol_label: p.protocol_label,
-        download:       p.download || 0,
-        upload:         p.upload || 0,
-        flows:          p.flows || 0,
-      }));
-      await ProtocolStat.insertMany(protoDocs);
-      console.log(`[Collector]   ✓ ${protoDocs.length} protocols saved for ${label}`);
-    }
-
-    // 2h. SNI Hostnames
-    const snis = await netify.fetchSniHostnames(5, 10000, agentUuid, SITE_UUID);
-    if (snis && snis.length > 0) {
-      const sniDocs = snis.map(s => ({
-        timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
-        sni_hostname: (s.sni_hostname && String(s.sni_hostname).trim() !== '') ? s.sni_hostname : 'Unknown',
-        download: s.download || 0, upload: s.upload || 0, flows: s.flows || 0,
-      }));
-      await SniHostnameStat.insertMany(sniDocs);
-      console.log(`[Collector]   ✓ ${sniDocs.length} SNI hostnames saved for ${label}`);
-    }
-
-    /* -- COMMENTED OUT DUE TO NETIFY API HTTP 422 (UNSUPPORTED TIER) --
-    // 2i. SSL Server Common Names
-    const cns = await netify.fetchSslServerCn(1440, 50, agentUuid);
-    if (cns && cns.length > 0) {
-      const cnDocs = cns.map(c => ({
-        timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
-        ssl_server_cn: c.ssl_server_cn, download: c.download || 0, upload: c.upload || 0, flows: c.flows || 0,
-      }));
-      await SslServerCnStat.insertMany(cnDocs);
-      console.log(`[Collector]   ✓ ${cnDocs.length} SSL Server CNs saved for ${label}`);
-    }
-
-    // 2j. QUIC Hostnames
-    const quics = await netify.fetchQuicHostnames(1440, 50, agentUuid);
-    if (quics && quics.length > 0) {
-      const quicDocs = quics.map(q => ({
-        timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
-        quic_hostname: q.quic_hostname, download: q.download || 0, upload: q.upload || 0, flows: q.flows || 0,
-      }));
-      await QuicHostnameStat.insertMany(quicDocs);
-      console.log(`[Collector]   ✓ ${quicDocs.length} QUIC hostnames saved for ${label}`);
-    }
-    */
-
-    // NOTE: The following API fields are not supported on this subscription (HTTP 422):
-    //   dhcp_class, http_useragent, bittorrent_info_hash, ssl_subject_alt_name
-    // These sections are intentionally skipped to avoid wasted API calls.
-    // Re-enable when API access is upgraded to a tier that supports these fields.
-
-  } catch (err) {
-    console.error(`[CollectorHelper] Error saving secondary telemetry:`, err.message);
-  }
-}
-
-module.exports = {
-  collectSecondaryTelemetry,
-};
+// proxy/collectorHelper.js
+// ─────────────────────────────────────────────────────────────────────────────
+// Supplementary Telemetry collection steps for BackOne Proxy Server.
+// Split from collector.js to satisfy the 256-line file size limit.
+// ─────────────────────────────────────────────────────────────────────────────
+
+const {
+  AppCategoryStat,
+  TlsVersionStat,
+  TlsCipherStat,
+  TlsSecurityStat,
+  CountryStat,
+  ProtocolStat,
+  SslSubjectAltNameStat,
+  SniHostnameStat,
+  SslServerCnStat,
+  QuicHostnameStat,
+} = require('./models/Schemas');
+
+async function collectSecondaryTelemetry(agentUuid, timestamp, SITE_UUID, backone, label) {
+  try {
+    // 2b. App Categories
+    const categories = await backone.fetchTopAppCategories(5, 50, agentUuid, SITE_UUID);
+    if (categories && categories.length > 0) {
+      const catDocs = categories.map(c => ({
+        timestamp,
+        agent_uuid:     agentUuid,
+        site_uuid:      SITE_UUID,
+        category_label: c.category_label,
+        download:       c.download || 0,
+        upload:         c.upload || 0,
+        flows:          c.flows || 0,
+      }));
+      await AppCategoryStat.insertMany(catDocs);
+      console.log(`[Collector]   ✓ ${catDocs.length} categories saved for ${label}`);
+    }
+
+    // 2c. TLS Versions
+    const tlsVersions = await backone.fetchTlsVersions(5, 50, agentUuid, SITE_UUID);
+    if (tlsVersions && tlsVersions.length > 0) {
+      const tvDocs = tlsVersions.map(v => ({
+        timestamp,
+        agent_uuid:   agentUuid,
+        site_uuid:    SITE_UUID,
+        tls_version:  v.tls_version,
+        download:     v.download || 0,
+        upload:       v.upload || 0,
+        flows:        v.flows || 0,
+      }));
+      await TlsVersionStat.insertMany(tvDocs);
+      console.log(`[Collector]   ✓ ${tvDocs.length} TLS versions saved for ${label}`);
+    }
+
+    // 2d. TLS Ciphers
+    const tlsCiphers = await backone.fetchTlsCiphers(5, 50, agentUuid, SITE_UUID);
+    if (tlsCiphers && tlsCiphers.length > 0) {
+      const tcDocs = tlsCiphers.map(c => ({
+        timestamp,
+        agent_uuid:   agentUuid,
+        site_uuid:    SITE_UUID,
+        tls_cipher:   c.tls_cipher,
+        download:     c.download || 0,
+        upload:       c.upload || 0,
+        flows:        c.flows || 0,
+      }));
+      await TlsCipherStat.insertMany(tcDocs);
+      console.log(`[Collector]   ✓ ${tcDocs.length} TLS ciphers saved for ${label}`);
+    }
+
+    // 2e. TLS Security
+    const tlsSecurity = await backone.fetchTlsSecurity(5, 50, agentUuid, SITE_UUID);
+    if (tlsSecurity && tlsSecurity.length > 0) {
+      const tsDocs = tlsSecurity.map(s => ({
+        timestamp,
+        agent_uuid:   agentUuid,
+        site_uuid:    SITE_UUID,
+        tls_security: s.tls_security,
+        download:     s.download || 0,
+        upload:       s.upload || 0,
+        flows:        s.flows || 0,
+      }));
+      await TlsSecurityStat.insertMany(tsDocs);
+      console.log(`[Collector]   ✓ ${tsDocs.length} TLS security stats saved for ${label}`);
+    }
+
+    // 2f. Top Countries
+    const countries = await backone.fetchTopCountries(5, 100, agentUuid, SITE_UUID);
+    if (countries && countries.length > 0) {
+      const coDocs = countries.map(c => ({
+        timestamp,
+        agent_uuid:   agentUuid,
+        site_uuid:    SITE_UUID,
+        country_code: c.country_code,
+        country_name: c.country_name || '',
+        download:     c.download || 0,
+        upload:       c.upload   || 0,
+        flows:        c.flows    || 0,
+      }));
+      await CountryStat.insertMany(coDocs);
+      console.log(`[Collector]   ✓ ${coDocs.length} countries saved for ${label}`);
+    }
+
+    // 2g. Top Protocols
+    const protocols = await backone.fetchTopProtocols(5, 50, agentUuid, SITE_UUID);
+    if (protocols && protocols.length > 0) {
+      const protoDocs = protocols.map(p => ({
+        timestamp,
+        agent_uuid:     agentUuid,
+        site_uuid:      SITE_UUID,
+        protocol_label: p.protocol_label,
+        download:       p.download || 0,
+        upload:         p.upload || 0,
+        flows:          p.flows || 0,
+      }));
+      await ProtocolStat.insertMany(protoDocs);
+      console.log(`[Collector]   ✓ ${protoDocs.length} protocols saved for ${label}`);
+    }
+
+    // 2h. SNI Hostnames
+    const snis = await backone.fetchSniHostnames(5, 10000, agentUuid, SITE_UUID);
+    if (snis && snis.length > 0) {
+      const sniDocs = snis.map(s => ({
+        timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
+        sni_hostname: (s.sni_hostname && String(s.sni_hostname).trim() !== '') ? s.sni_hostname : 'Unknown',
+        download: s.download || 0, upload: s.upload || 0, flows: s.flows || 0,
+      }));
+      await SniHostnameStat.insertMany(sniDocs);
+      console.log(`[Collector]   ✓ ${sniDocs.length} SNI hostnames saved for ${label}`);
+    }
+
+    /* -- COMMENTED OUT DUE TO BACKONE API HTTP 422 (UNSUPPORTED TIER) --
+    // 2i. SSL Server Common Names
+    const cns = await backone.fetchSslServerCn(1440, 50, agentUuid);
+    if (cns && cns.length > 0) {
+      const cnDocs = cns.map(c => ({
+        timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
+        ssl_server_cn: c.ssl_server_cn, download: c.download || 0, upload: c.upload || 0, flows: c.flows || 0,
+      }));
+      await SslServerCnStat.insertMany(cnDocs);
+      console.log(`[Collector]   ✓ ${cnDocs.length} SSL Server CNs saved for ${label}`);
+    }
+
+    // 2j. QUIC Hostnames
+    const quics = await backone.fetchQuicHostnames(1440, 50, agentUuid);
+    if (quics && quics.length > 0) {
+      const quicDocs = quics.map(q => ({
+        timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
+        quic_hostname: q.quic_hostname, download: q.download || 0, upload: q.upload || 0, flows: q.flows || 0,
+      }));
+      await QuicHostnameStat.insertMany(quicDocs);
+      console.log(`[Collector]   ✓ ${quicDocs.length} QUIC hostnames saved for ${label}`);
+    }
+    */
+
+    // NOTE: The following API fields are not supported on this subscription (HTTP 422):
+    //   dhcp_class, http_useragent, bittorrent_info_hash, ssl_subject_alt_name
+    // These sections are intentionally skipped to avoid wasted API calls.
+    // Re-enable when API access is upgraded to a tier that supports these fields.
+
+  } catch (err) {
+    console.error(`[CollectorHelper] Error saving secondary telemetry:`, err.message);
+  }
+}
+
+module.exports = {
+  collectSecondaryTelemetry,
+};

proxy/collectorHelperDpi2.js

@@ -13,8 +13,8 @@ const {
   generateAutoLabel
 } = require('./deviceResolver');
 
-async function collectDevicesAndApps(agentUuid, timestamp, SITE_UUID, netify, label) {
-  const devices = await netify.fetchDiscoveredDevices(5, 500, agentUuid, SITE_UUID);
+async function collectDevicesAndApps(agentUuid, timestamp, SITE_UUID, backone, label) {
+  const devices = await backone.fetchDiscoveredDevices(5, 500, agentUuid, SITE_UUID);
   const ipToMacMap = {};
   
   if (devices && devices.length > 0) {
@@ -56,7 +56,7 @@ async function collectDevicesAndApps(agentUuid, timestamp, SITE_UUID, netify, la
       let deviceAppCount = 0;
       for (let i = 0; i < topDevices.length; i += 5) {
         const batch = topDevices.slice(i, i + 5);
-        const results = await Promise.allSettled(batch.map(d => netify.fetchDeviceApps(d.ip_address, 5, 50, agentUuid, SITE_UUID)));
+        const results = await Promise.allSettled(batch.map(d => backone.fetchDeviceApps(d.ip_address, 5, 50, agentUuid, SITE_UUID)));
         const appDocs = [];
         results.forEach((res, idx) => {
           if (res.status === 'fulfilled' && Array.isArray(res.value)) {
@@ -80,10 +80,13 @@ async function collectDevicesAndApps(agentUuid, timestamp, SITE_UUID, netify, la
   return ipToMacMap;
 }
 
-async function collectFlows(agentUuid, timestamp, SITE_UUID, netify, label, ipToMacMap) {
-  // Netify API has a hard limit of 1,000,000 for settings_limit. Use 1000000 as default per rule.
+async function collectFlows(agentUuid, timestamp, SITE_UUID, backone, label, ipToMacMap) {
+  // BackOne API has a limit of 1,000,000 for settings_limit. Use 1000000 as default per rule.
   const flowLimit = parseInt(process.env.PROXY_FLOW_LIMIT || '1000000');
-  const flows = await netify.fetchFlows(flowLimit, agentUuid, SITE_UUID);
+  // Fetch flows dari 24 jam terakhir (1440 menit) agar semua flow — termasuk yang sudah selesai —
+  // tersimpan ke MongoDB, bukan hanya koneksi yang aktif saat ini.
+  const FLOW_INTERVAL_MINUTES = 1440;
+  const flows = await backone.fetchFlows(flowLimit, agentUuid, SITE_UUID, FLOW_INTERVAL_MINUTES);
   if (flows && flows.length > 0) {
     const flowDocs = flows.map(f => {
       const mac = f.src_mac || ipToMacMap[f.src_ip] || generateMacFromIp(f.src_ip);

proxy/collectorHelperDpi3.js

@@ -6,8 +6,8 @@
 
 const { DeviceStat, Threat, Event } = require('./models/Schemas');
 
-async function collectThreats(agentUuid, timestamp, SITE_UUID, netify, label) {
-  const threats = await netify.fetchCyberThreats(agentUuid, SITE_UUID);
+async function collectThreats(agentUuid, timestamp, SITE_UUID, backone, label) {
+  const threats = await backone.fetchCyberThreats(agentUuid, SITE_UUID);
   if (threats && threats.length > 0) {
     const threatDocs = threats.map(t => ({
       timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
@@ -20,8 +20,8 @@ async function collectThreats(agentUuid, timestamp, SITE_UUID, netify, label) {
   }
 }
 
-async function collectEvents(agentUuid, timestamp, SITE_UUID, netify, label) {
-  const events = await netify.fetchEvents(100, agentUuid, SITE_UUID);
+async function collectEvents(agentUuid, timestamp, SITE_UUID, backone, label) {
+  const events = await backone.fetchEvents(100, agentUuid, SITE_UUID);
   if (events && events.length > 0) {
     const eventIds = events.map(e => e.event_id).filter(id => id !== null);
     const existing = await Event.find({ site_uuid: SITE_UUID, event_id: { $in: eventIds } }).distinct('event_id');

proxy/dataRetention.js

@@ -1,22 +1,22 @@
 // proxy/dataRetention.js
 // ─────────────────────────────────────────────────────────────────────────────
 // Pruning process for BackOne MongoDB data retention.
-// Removes telemetry records older than 7 days to conserve database space.
+// Removes telemetry records older than 30 days to conserve database space.
 // ─────────────────────────────────────────────────────────────────────────────
 
 const mongoose = require('mongoose');
 const Schemas = require('./models/Schemas');
 
 /**
- * Prune all time-series documents older than 7 days.
+ * Prune all time-series documents older than 30 days.
  */
 async function pruneOldData() {
   const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
   const timeString = thirtyDaysAgo.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
   console.log(`[Collector] [Retention] Checking for telemetry data older than 30 days (before ${timeString})...`);
 
-  // Prune from all collections in Schemas except CustomDeviceLabel
-  const collections = Object.keys(Schemas).filter(name => name !== 'CustomDeviceLabel');
+  // Prune from all collections in Schemas except CustomDeviceLabel, AgentRegistry, TenantConfig
+  const collections = Object.keys(Schemas).filter(name => name !== 'CustomDeviceLabel' && name !== 'AgentRegistry' && name !== 'TenantConfig');
 
   for (const name of collections) {
     try {

proxy/models/Schemas.js

@@ -1,199 +1,198 @@
-// proxy/models/Schemas.js
-// MongoDB schemas shared between the proxy server (write) and backend (read).
-// Each document is tagged with agent_uuid + site_uuid for tenant isolation.
-//
-// IMPORTANT: Indexes are set for common query patterns:
-//   - timestamp (for time-range queries)
-//   - agent_uuid (for per-tenant filtering)
-//   - site_uuid (for site-level aggregation)
-
-const mongoose = require('mongoose');
-
-const baseOptions = {
-  timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
-};
-
-// ─── Bandwidth Summary (per agent, per collection cycle) ───────────────────────
-const SummarySchema = new mongoose.Schema({
-  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid:     { type: String, index: true },  // null = global/all agents
-  site_uuid:      { type: String, index: true },
-  bandwidth_down: Number,
-  bandwidth_up:   Number,
-  active_flows:   Number,
-  download_speed: Number,
-  upload_speed:   Number,
-  total_devices:  Number,
-  total_threats:  Number,
-  packet_drops:   Number,
-  peak_flow_rate: Number,
-  cpu_usage:      Number,
-  memory_usage:   Number,
-  queue_depth:    Number,
-}, baseOptions);
-
-// ─── Top Applications (per agent) ─────────────────────────────────────────────
-const AppStatSchema = new mongoose.Schema({
-  timestamp:  { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid: { type: String, index: true },
-  site_uuid:  { type: String, index: true },
-  app_label:  { type: String, required: true },
-  download:   Number,
-  upload:     Number,
-  flows:      Number,
-}, baseOptions);
-
-// ─── Protocol Statistics (per agent) ──────────────────────────────────────────
-const ProtocolStatSchema = new mongoose.Schema({
-  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid:     { type: String, index: true },
-  site_uuid:      { type: String, index: true },
-  protocol_label: { type: String, required: true },
-  download:       Number,
-  upload:         Number,
-  flows:          Number,
-}, baseOptions);
-
-// ─── Discovered Devices (per agent, includes IP + MAC + device info) ───────────
-const DeviceStatSchema = new mongoose.Schema({
-  timestamp:    { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid:   { type: String, index: true },
-  site_uuid:    { type: String, index: true },
-  ip_address:   { type: String, required: true, index: true },
-  mac_address:  { type: String, index: true },
-  device_label: String,
-  device_type:  String,
-  os_label:     String,
-  manufacturer: String,
-  download:     Number,
-  upload:       Number,
-  flows:        Number,
-  last_seen:    String,
-}, baseOptions);
-
-// ─── Network Flows (per agent) ─────────────────────────────────────────────────
-const FlowSchema = new mongoose.Schema({
-  timestamp:  { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid: { type: String, index: true },
-  site_uuid:  { type: String, index: true },
-  flow_id:    String,
-  src_ip:     { type: String, index: true },
-  src_mac:    { type: String, index: true },  // indexed for MAC-to-IP resolution in events
-  dst_ip:     { type: String, index: true },
-  dst_port:   Number,
-  protocol:   String,
-  app_label:  String,
-  domain:     String,
-  download:   Number,
-  upload:     Number,
-  first_seen: String,
-  last_seen:  String,
-}, baseOptions);
-
-// ─── Cyber Threats (per agent) ─────────────────────────────────────────────────
-const ThreatSchema = new mongoose.Schema({
-  timestamp:   { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid:  { type: String, index: true },
-  site_uuid:   { type: String, index: true },
-  threat_type: String,
-  severity:    String,
-  src_ip:      String,
-  dst_ip:      String,
-  dst_port:    Number,
-  protocol:    String,
-  description: String,
-  event_at:    String,
-  flow_id:     { type: String, index: true },
-}, baseOptions);
-
-// ─── App Categories (per agent) ───────────────────────────────────────────────
-const AppCategoryStatSchema = new mongoose.Schema({
-  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid:     { type: String, index: true },
-  site_uuid:      { type: String, index: true },
-  category_label: { type: String, required: true },
-  download:       Number,
-  upload:         Number,
-  flows:          Number,
-}, baseOptions);
-
-// ─── System Events (per agent) ─────────────────────────────────────────────────
-const EventSchema = new mongoose.Schema({
-  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid:     { type: String, index: true },
-  site_uuid:      { type: String, index: true },
-  event_id:       Number,
-  event_type:     String,
-  severity:       String,
-  description:    String,
-  category_label: String,
-  ip_address:     String,
-  mac_address:    String,
-  event_at:       Date,
-  flow_id:        { type: String, index: true },
-}, baseOptions);
-
-// ─── Compound indexes for common dashboard queries ─────────────────────────────
-SummarySchema.index({ agent_uuid: 1, timestamp: -1 });
-AppStatSchema.index({ agent_uuid: 1, timestamp: -1, download: -1 });
-DeviceStatSchema.index({ agent_uuid: 1, ip_address: 1 }, { unique: true });
-FlowSchema.index({ agent_uuid: 1, timestamp: -1 });
-FlowSchema.index({ agent_uuid: 1, flow_id: 1 });
-FlowSchema.index({ site_uuid: 1, timestamp: -1 });
-FlowSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
-ThreatSchema.index({ agent_uuid: 1, timestamp: -1 });
-AppCategoryStatSchema.index({ agent_uuid: 1, timestamp: -1 });
-EventSchema.index({ agent_uuid: 1, timestamp: -1 });
-FlowSchema.index({ site_uuid: 1, src_mac: 1, timestamp: -1 }); // for MAC-to-IP resolution
-
-// ── Per-Device Per-Application Stats ────────────────────────────────────────
-// Collected from DPI API: /data/stats/top/application/download with filter_local_ips
-// Allows showing "YouTube 134GB" in Device Detail modal per specific IP
-const DeviceAppStatSchema = new mongoose.Schema({
-  timestamp:  { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid: { type: String, index: true },
-  site_uuid:  { type: String, index: true },
-  ip_address: { type: String, required: true, index: true },
-  app_label:  { type: String, required: true },
-  app_id:     Number,
-  download:   { type: Number, default: 0 },
-  upload:     { type: Number, default: 0 },
-  flows:      { type: Number, default: 0 },
-  last_seen:  String,
-}, baseOptions);
-DeviceAppStatSchema.index({ agent_uuid: 1, ip_address: 1, timestamp: -1 });
-DeviceAppStatSchema.index({ ip_address: 1, app_label: 1, timestamp: -1 });
-DeviceAppStatSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
-
-const telemetrySchemas = require('./SchemasTelemetry');
-const auxSchemas = require('./SchemasAux');
-
-// ─── Agent Registry (all agents registered in Netify, regardless of activity) ──
-// Upserted every collector cycle. Source of truth for the agents list page.
-const AgentRegistrySchema = new mongoose.Schema({
-  uuid:         { type: String, required: true, unique: true, index: true },
-  serial:       { type: String },
-  label:        { type: String },
-  site_uuid:    { type: String, index: true },
-  provisioned:  { type: Boolean, default: false },
-  activated:    { type: Boolean, default: false },
-  last_seen_at: { type: mongoose.Schema.Types.Mixed },
-  netify_id:    { type: Number },
-}, { ...baseOptions, collection: 'agent_registry' });
-
-module.exports = {
-  Summary:       mongoose.model('Summary',       SummarySchema),
-  AppStat:       mongoose.model('AppStat',       AppStatSchema),
-  ProtocolStat:  mongoose.model('ProtocolStat',  ProtocolStatSchema),
-  DeviceStat:    mongoose.model('DeviceStat',    DeviceStatSchema),
-  DeviceAppStat: mongoose.model('DeviceAppStat', DeviceAppStatSchema),
-  Flow:          mongoose.model('Flow',          FlowSchema),
-  Threat:        mongoose.model('Threat',        ThreatSchema),
-  AppCategoryStat: mongoose.model('AppCategoryStat', AppCategoryStatSchema),
-  Event:         mongoose.model('Event',         EventSchema),
-  AgentRegistry: mongoose.model('AgentRegistry', AgentRegistrySchema),
-  ...auxSchemas,
-  ...telemetrySchemas,
-};
-
-
+// proxy/models/Schemas.js
+// MongoDB schemas shared between the proxy server (write) and backend (read).
+// Each document is tagged with agent_uuid + site_uuid for tenant isolation.
+//
+// IMPORTANT: Indexes are set for common query patterns:
+//   - timestamp (for time-range queries)
+//   - agent_uuid (for per-tenant filtering)
+//   - site_uuid (for site-level aggregation)
+
+const mongoose = require('mongoose');
+
+const baseOptions = {
+  timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
+};
+
+// ─── Bandwidth Summary (per agent, per collection cycle) ───────────────────────
+const SummarySchema = new mongoose.Schema({
+  timestamp:      { type: Date,   required: true, index: true, expires: '30d' },
+  agent_uuid:     { type: String, index: true },  // null = global/all agents
+  site_uuid:      { type: String, index: true },
+  bandwidth_down: Number,
+  bandwidth_up:   Number,
+  active_flows:   Number,
+  download_speed: Number,
+  upload_speed:   Number,
+  total_devices:  Number,
+  total_threats:  Number,
+  packet_drops:   Number,
+  peak_flow_rate: Number,
+  cpu_usage:      Number,
+  memory_usage:   Number,
+  queue_depth:    Number,
+}, baseOptions);
+
+// ─── Top Applications (per agent) ─────────────────────────────────────────────
+const AppStatSchema = new mongoose.Schema({
+  timestamp:  { type: Date,   required: true, index: true, expires: '30d' },
+  agent_uuid: { type: String, index: true },
+  site_uuid:  { type: String, index: true },
+  app_label:  { type: String, required: true },
+  download:   Number,
+  upload:     Number,
+  flows:      Number,
+}, baseOptions);
+
+// ─── Protocol Statistics (per agent) ──────────────────────────────────────────
+const ProtocolStatSchema = new mongoose.Schema({
+  timestamp:      { type: Date,   required: true, index: true, expires: '30d' },
+  agent_uuid:     { type: String, index: true },
+  site_uuid:      { type: String, index: true },
+  protocol_label: { type: String, required: true },
+  download:       Number,
+  upload:         Number,
+  flows:          Number,
+}, baseOptions);
+
+// ─── Discovered Devices (per agent, includes IP + MAC + device info) ───────────
+const DeviceStatSchema = new mongoose.Schema({
+  timestamp:    { type: Date,   required: true, index: true, expires: '30d' },
+  agent_uuid:   { type: String, index: true },
+  site_uuid:    { type: String, index: true },
+  ip_address:   { type: String, required: true, index: true },
+  mac_address:  { type: String, index: true },
+  device_label: String,
+  device_type:  String,
+  os_label:     String,
+  manufacturer: String,
+  download:     Number,
+  upload:       Number,
+  flows:        Number,
+  last_seen:    String,
+}, baseOptions);
+
+// ─── Network Flows (per agent) ─────────────────────────────────────────────────
+const FlowSchema = new mongoose.Schema({
+  timestamp:  { type: Date,   required: true, index: true, expires: '30d' },
+  agent_uuid: { type: String, index: true },
+  site_uuid:  { type: String, index: true },
+  flow_id:    String,
+  src_ip:     { type: String, index: true },
+  src_mac:    { type: String, index: true },  // indexed for MAC-to-IP resolution in events
+  dst_ip:     { type: String, index: true },
+  dst_port:   Number,
+  protocol:   String,
+  app_label:  String,
+  domain:     String,
+  download:   Number,
+  upload:     Number,
+  first_seen: String,
+  last_seen:  String,
+}, baseOptions);
+
+// ─── Cyber Threats (per agent) ─────────────────────────────────────────────────
+const ThreatSchema = new mongoose.Schema({
+  timestamp:   { type: Date,   required: true, index: true, expires: '30d' },
+  agent_uuid:  { type: String, index: true },
+  site_uuid:   { type: String, index: true },
+  threat_type: String,
+  severity:    String,
+  src_ip:      String,
+  dst_ip:      String,
+  dst_port:    Number,
+  protocol:    String,
+  description: String,
+  event_at:    String,
+  flow_id:     { type: String, index: true },
+}, baseOptions);
+
+// ─── App Categories (per agent) ───────────────────────────────────────────────
+const AppCategoryStatSchema = new mongoose.Schema({
+  timestamp:      { type: Date,   required: true, index: true, expires: '30d' },
+  agent_uuid:     { type: String, index: true },
+  site_uuid:      { type: String, index: true },
+  category_label: { type: String, required: true },
+  download:       Number,
+  upload:         Number,
+  flows:          Number,
+}, baseOptions);
+
+// ─── System Events (per agent) ─────────────────────────────────────────────────
+const EventSchema = new mongoose.Schema({
+  timestamp:      { type: Date,   required: true, index: true, expires: '30d' },
+  agent_uuid:     { type: String, index: true },
+  site_uuid:      { type: String, index: true },
+  event_id:       Number,
+  event_type:     String,
+  severity:       String,
+  description:    String,
+  category_label: String,
+  ip_address:     String,
+  mac_address:    String,
+  event_at:       Date,
+  flow_id:        { type: String, index: true },
+}, baseOptions);
+
+// ─── Compound indexes for common dashboard queries ─────────────────────────────
+SummarySchema.index({ agent_uuid: 1, timestamp: -1 });
+AppStatSchema.index({ agent_uuid: 1, timestamp: -1, download: -1 });
+DeviceStatSchema.index({ agent_uuid: 1, ip_address: 1 }, { unique: true });
+FlowSchema.index({ agent_uuid: 1, timestamp: -1 });
+FlowSchema.index({ agent_uuid: 1, flow_id: 1 });
+FlowSchema.index({ site_uuid: 1, timestamp: -1 });
+FlowSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
+ThreatSchema.index({ agent_uuid: 1, timestamp: -1 });
+AppCategoryStatSchema.index({ agent_uuid: 1, timestamp: -1 });
+EventSchema.index({ agent_uuid: 1, timestamp: -1 });
+FlowSchema.index({ site_uuid: 1, src_mac: 1, timestamp: -1 }); // for MAC-to-IP resolution
+
+// ── Per-Device Per-Application Stats ────────────────────────────────────────
+// Collected from DPI API: /data/stats/top/application/download with filter_local_ips
+// Allows showing "YouTube 134GB" in Device Detail modal per specific IP
+const DeviceAppStatSchema = new mongoose.Schema({
+  timestamp:  { type: Date,   required: true, index: true, expires: '30d' },
+  agent_uuid: { type: String, index: true },
+  site_uuid:  { type: String, index: true },
+  ip_address: { type: String, required: true, index: true },
+  app_label:  { type: String, required: true },
+  app_id:     Number,
+  download:   { type: Number, default: 0 },
+  upload:     { type: Number, default: 0 },
+  flows:      { type: Number, default: 0 },
+  last_seen:  String,
+}, baseOptions);
+DeviceAppStatSchema.index({ agent_uuid: 1, ip_address: 1, timestamp: -1 });
+DeviceAppStatSchema.index({ ip_address: 1, app_label: 1, timestamp: -1 });
+DeviceAppStatSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
+
+const telemetrySchemas = require('./SchemasTelemetry');
+const auxSchemas = require('./SchemasAux');
+
+// ─── Agent Registry (all agents registered in BackOne, regardless of activity) ──
+// Upserted every collector cycle. Source of truth for the agents list page.
+const AgentRegistrySchema = new mongoose.Schema({
+  uuid:         { type: String, required: true, unique: true, index: true },
+  serial:       { type: String },
+  label:        { type: String },
+  site_uuid:    { type: String, index: true },
+  provisioned:  { type: Boolean, default: false },
+  activated:    { type: Boolean, default: false },
+  last_seen_at: { type: mongoose.Schema.Types.Mixed },
+}, { ...baseOptions, collection: 'agent_registry' });
+
+module.exports = {
+  Summary:       mongoose.model('Summary',       SummarySchema),
+  AppStat:       mongoose.model('AppStat',       AppStatSchema),
+  ProtocolStat:  mongoose.model('ProtocolStat',  ProtocolStatSchema),
+  DeviceStat:    mongoose.model('DeviceStat',    DeviceStatSchema),
+  DeviceAppStat: mongoose.model('DeviceAppStat', DeviceAppStatSchema),
+  Flow:          mongoose.model('Flow',          FlowSchema),
+  Threat:        mongoose.model('Threat',        ThreatSchema),
+  AppCategoryStat: mongoose.model('AppCategoryStat', AppCategoryStatSchema),
+  Event:         mongoose.model('Event',         EventSchema),
+  AgentRegistry: mongoose.model('AgentRegistry', AgentRegistrySchema),
+  ...auxSchemas,
+  ...telemetrySchemas,
+};
+
+

proxy/models/SchemasAux.js

@@ -11,7 +11,7 @@ const baseOptions = {
 
 // ─── TLS Versions (per agent) ──────────────────────────────────────────────────
 const TlsVersionStatSchema = new mongoose.Schema({
-  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
+  timestamp:      { type: Date,   required: true, index: true, expires: '30d' },
   agent_uuid:     { type: String, index: true },
   site_uuid:      { type: String, index: true },
   tls_version:    { type: String, required: true },
@@ -22,7 +22,7 @@ const TlsVersionStatSchema = new mongoose.Schema({
 
 // ─── TLS Ciphers (per agent) ───────────────────────────────────────────────────
 const TlsCipherStatSchema = new mongoose.Schema({
-  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
+  timestamp:      { type: Date,   required: true, index: true, expires: '30d' },
   agent_uuid:     { type: String, index: true },
   site_uuid:      { type: String, index: true },
   tls_cipher:     { type: String, required: true },
@@ -33,7 +33,7 @@ const TlsCipherStatSchema = new mongoose.Schema({
 
 // ─── TLS Security (per agent) ──────────────────────────────────────────────────
 const TlsSecurityStatSchema = new mongoose.Schema({
-  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
+  timestamp:      { type: Date,   required: true, index: true, expires: '30d' },
   agent_uuid:     { type: String, index: true },
   site_uuid:      { type: String, index: true },
   tls_security:   { type: String, required: true },
@@ -44,7 +44,7 @@ const TlsSecurityStatSchema = new mongoose.Schema({
 
 // ─── Country Traffic Stats (per agent) ────────────────────────────────────────
 const CountryStatSchema = new mongoose.Schema({
-  timestamp:    { type: Date,   required: true, index: true, expires: '7d' },
+  timestamp:    { type: Date,   required: true, index: true, expires: '30d' },
   agent_uuid:   { type: String, index: true },
   site_uuid:    { type: String, index: true },
   country_code: { type: String, required: true },

proxy/models/SchemasTelemetry.js

@@ -1,7 +1,7 @@
 // proxy/models/SchemasTelemetry.js
 // ─────────────────────────────────────────────────────────────────────────────
 // DPI Telemetry Property Schemas — Split from Schemas.js to stay under 256 lines.
-// These are Netify-specific data fields collected via /data/stats/top/<field> API.
+// These are BackOne-specific data fields collected via /data/stats/top/<field> API.
 // ─────────────────────────────────────────────────────────────────────────────
 
 const mongoose = require('mongoose');
@@ -13,7 +13,7 @@ const baseOptions = {
 // ─── Helper: build a consistent DPI property schema ───────────────────────────
 function dpiPropertySchema(fieldName) {
   const fields = {
-    timestamp:  { type: Date,   required: true, index: true, expires: '7d' },
+    timestamp:  { type: Date,   required: true, index: true, expires: '30d' },
     agent_uuid: { type: String, index: true },
     site_uuid:  { type: String, index: true },
     download:   Number,
@@ -34,7 +34,7 @@ const HttpUserAgentStatSchema = dpiPropertySchema('user_agent');
 
 // ─── BitTorrent Info Hashes (bittorrent_info_hash) ────────────────────────────
 const BittorrentHashStatSchema = new mongoose.Schema({
-  timestamp:  { type: Date,   required: true, index: true, expires: '7d' },
+  timestamp:  { type: Date,   required: true, index: true, expires: '30d' },
   agent_uuid: { type: String, index: true },
   site_uuid:  { type: String, index: true },
   info_hash:  { type: String, required: true },

proxy/package.json

@@ -1,19 +1,19 @@
-{
-  "name": "backone-proxy",
-  "version": "1.0.0",
-  "description": "BackOne DPI Proxy Server - Fetches from DPI API, filters per agent_uuid, stores to MongoDB",
-  "main": "index.js",
-  "scripts": {
-    "start": "node index.js",
-    "start:bun": "bun run index.js",
-    "dev": "nodemon index.js"
-  },
-  "dependencies": {
-    "axios": "^1.6.2",
-    "cors": "^2.8.5",
-    "dotenv": "^16.3.1",
-    "express": "^4.18.2",
-    "mongoose": "^8.0.3",
-    "node-cron": "^3.0.3"
-  }
-}
+{
+  "name": "backone-proxy",
+  "version": "1.0.0",
+  "description": "BackOne DPI Proxy Server - Fetches from DPI API, filters per agent_uuid, stores to MongoDB",
+  "main": "index.js",
+  "scripts": {
+    "start": "node index.js",
+    "start:bun": "bun run index.js",
+    "dev": "nodemon index.js"
+  },
+  "dependencies": {
+    "axios": "^1.6.2",
+    "cors": "^2.8.5",
+    "dotenv": "^16.3.1",
+    "express": "^4.18.2",
+    "mongoose": "^8.0.3",
+    "node-cron": "^3.0.3"
+  }
+}

proxy/scheduler.js

@@ -1,129 +1,129 @@
-// proxy/scheduler.js
-// Cron scheduler for automatic data collection from DPI API
-// Runs every 5 minutes, collecting data for all agents or a specific agent.
-
-const cron = require('node-cron');
-const { collectAllAgents, collectSpecificAgent, collectSpecificAgents } = require('./collector');
-const { syncApplicationDictionary } = require('./netifyClient');
-
-// Mode: 'all' = collect all agents, 'agent' = collect one specific agent, 'agents' = collect multiple agents
-const COLLECT_MODE    = process.env.PROXY_COLLECT_MODE || 'all';
-const SPECIFIC_AGENT  = process.env.PROXY_AGENT_UUID   || null;
-const SPECIFIC_AGENTS = (process.env.PROXY_AGENT_UUIDS || '').split(',').map(s => s.trim()).filter(Boolean);
-const AGENT_DELAY_MS  = parseInt(process.env.PROXY_AGENT_DELAY_MS || '5000');
-const CRON_SCHEDULE   = process.env.PROXY_CRON_SCHEDULE || '*/5 * * * *';
-
-// Capacity logging is an expensive full-scan aggregation. Run it at most once per
-// interval (default 24h) instead of every collection cycle to reduce CPU/DB load.
-const CAPACITY_LOG_INTERVAL_MS = parseInt(process.env.PROXY_CAPACITY_LOG_INTERVAL_MS || String(24 * 60 * 60 * 1000));
-
-let isRunning = false;
-let lastRunAt = null;
-let lastRunResult = null;
-let runCount = 0;
-let lastCapacityLogAt = 0;
-
-/**
- * Execute one collection cycle (called by cron and manual trigger).
- * Prevents concurrent runs with isRunning guard.
- */
-async function runCollection() {
-  if (isRunning) {
-    console.log('[Scheduler] Skipping - previous run still in progress');
-    return { skipped: true, reason: 'already_running' };
-  }
-
-  isRunning = true;
-  lastRunAt = new Date();
-  runCount++;
-
-  try {
-    let result;
-    if (COLLECT_MODE === 'agent' && SPECIFIC_AGENT) {
-      console.log(`[Scheduler] Run #${runCount} - Mode: SPECIFIC AGENT (${SPECIFIC_AGENT})`);
-      result = await collectSpecificAgent(SPECIFIC_AGENT);
-    } else if (COLLECT_MODE === 'agents' && SPECIFIC_AGENTS.length > 0) {
-      console.log(`[Scheduler] Run #${runCount} - Mode: SPECIFIC AGENTS (${SPECIFIC_AGENTS.join(', ')})`);
-      result = await collectSpecificAgents(SPECIFIC_AGENTS, AGENT_DELAY_MS);
-    } else {
-      console.log(`[Scheduler] Run #${runCount} - Mode: ALL AGENTS`);
-      result = await collectAllAgents();
-    }
-    lastRunResult = { ...result, run_count: runCount };
-    
-    // Log MongoDB database capacity usage (expensive full-scan aggregation).
-    // Only run periodically (default: every 24h) to avoid high CPU/DB load each cycle.
-    const now = Date.now();
-    if (now - lastCapacityLogAt >= CAPACITY_LOG_INTERVAL_MS) {
-      lastCapacityLogAt = now;
-      const { logCapacityStats } = require('./db/capacityTracker');
-      await logCapacityStats(`[PROXY] [MongoDB] Capacity Used after Run #${runCount}:`);
-    }
-    
-    return lastRunResult;
-  } catch (err) {
-    console.error('[Scheduler] Unhandled error during collection:', err.message);
-    lastRunResult = { success: false, error: err.message, run_count: runCount };
-    return lastRunResult;
-  } finally {
-    isRunning = false;
-  }
-}
-
-/**
- * Start the scheduler (cron job + immediate first run).
- */
-function startScheduler() {
-  console.log(`[Scheduler] Starting proxy data collector`);
-  const modeLabel = COLLECT_MODE === 'agent'
-    ? `SPECIFIC AGENT (${SPECIFIC_AGENT})`
-    : COLLECT_MODE === 'agents'
-    ? `SPECIFIC AGENTS (${SPECIFIC_AGENTS.join(', ')})`
-    : 'ALL AGENTS';
-  console.log(`[Scheduler] Mode     : ${modeLabel}`);
-  console.log(`[Scheduler] Schedule : ${CRON_SCHEDULE} (every 5 minutes by default)`);
-
-  // Validate cron expression
-  if (!cron.validate(CRON_SCHEDULE)) {
-    console.error(`[Scheduler] Invalid cron expression: "${CRON_SCHEDULE}". Using default.`);
-  }
-
-  // Start recurring cron job
-  cron.schedule(CRON_SCHEDULE, () => {
-    runCollection().catch(err => console.error('[Scheduler] Cron error:', err.message));
-  });
-
-  console.log('[Scheduler] Cron job registered. Starting initial collection...');
-  
-  // Initial run immediately on startup (async, do not block server start)
-  setTimeout(async () => {
-    // 1. Sync dictionary first
-    try {
-      await syncApplicationDictionary();
-    } catch (err) {
-      console.error('[Scheduler] Error syncing application dictionary:', err.message);
-    }
-    
-    // 2. Start normal telemetry collection
-    runCollection().catch(err => console.error('[Scheduler] Initial run error:', err.message));
-  }, 2000);
-}
-
-/**
- * Get current scheduler status (for REST API endpoint).
- */
-function getStatus() {
-  return {
-    is_running:     isRunning,
-    run_count:      runCount,
-    last_run_at:    lastRunAt?.toISOString() ?? null,
-    collect_mode:   COLLECT_MODE,
-    agent_uuid:     SPECIFIC_AGENT,
-    agent_uuids:    COLLECT_MODE === 'agents' ? SPECIFIC_AGENTS : [],
-    agent_delay_ms: AGENT_DELAY_MS,
-    cron_schedule:  CRON_SCHEDULE,
-    last_result:    lastRunResult,
-  };
-}
-
-module.exports = { startScheduler, runCollection, getStatus };
+// proxy/scheduler.js
+// Cron scheduler for automatic data collection from DPI API
+// Runs every 5 minutes, collecting data for all agents or a specific agent.
+
+const cron = require('node-cron');
+const { collectAllAgents, collectSpecificAgent, collectSpecificAgents } = require('./collector');
+const { syncApplicationDictionary } = require('./backoneClient');
+
+// Mode: 'all' = collect all agents, 'agent' = collect one specific agent, 'agents' = collect multiple agents
+const COLLECT_MODE    = process.env.PROXY_COLLECT_MODE || 'all';
+const SPECIFIC_AGENT  = process.env.PROXY_AGENT_UUID   || null;
+const SPECIFIC_AGENTS = (process.env.PROXY_AGENT_UUIDS || '').split(',').map(s => s.trim()).filter(Boolean);
+const AGENT_DELAY_MS  = parseInt(process.env.PROXY_AGENT_DELAY_MS || '5000');
+const CRON_SCHEDULE   = process.env.PROXY_CRON_SCHEDULE || '*/5 * * * *';
+
+// Capacity logging is an expensive full-scan aggregation. Run it at most once per
+// interval (default 24h) instead of every collection cycle to reduce CPU/DB load.
+const CAPACITY_LOG_INTERVAL_MS = parseInt(process.env.PROXY_CAPACITY_LOG_INTERVAL_MS || String(24 * 60 * 60 * 1000));
+
+let isRunning = false;
+let lastRunAt = null;
+let lastRunResult = null;
+let runCount = 0;
+let lastCapacityLogAt = 0;
+
+/**
+ * Execute one collection cycle (called by cron and manual trigger).
+ * Prevents concurrent runs with isRunning guard.
+ */
+async function runCollection() {
+  if (isRunning) {
+    console.log('[Scheduler] Skipping - previous run still in progress');
+    return { skipped: true, reason: 'already_running' };
+  }
+
+  isRunning = true;
+  lastRunAt = new Date();
+  runCount++;
+
+  try {
+    let result;
+    if (COLLECT_MODE === 'agent' && SPECIFIC_AGENT) {
+      console.log(`[Scheduler] Run #${runCount} - Mode: SPECIFIC AGENT (${SPECIFIC_AGENT})`);
+      result = await collectSpecificAgent(SPECIFIC_AGENT);
+    } else if (COLLECT_MODE === 'agents' && SPECIFIC_AGENTS.length > 0) {
+      console.log(`[Scheduler] Run #${runCount} - Mode: SPECIFIC AGENTS (${SPECIFIC_AGENTS.join(', ')})`);
+      result = await collectSpecificAgents(SPECIFIC_AGENTS, AGENT_DELAY_MS);
+    } else {
+      console.log(`[Scheduler] Run #${runCount} - Mode: ALL AGENTS`);
+      result = await collectAllAgents();
+    }
+    lastRunResult = { ...result, run_count: runCount };
+    
+    // Log MongoDB database capacity usage (expensive full-scan aggregation).
+    // Only run periodically (default: every 24h) to avoid high CPU/DB load each cycle.
+    const now = Date.now();
+    if (now - lastCapacityLogAt >= CAPACITY_LOG_INTERVAL_MS) {
+      lastCapacityLogAt = now;
+      const { logCapacityStats } = require('./db/capacityTracker');
+      await logCapacityStats(`[PROXY] [MongoDB] Capacity Used after Run #${runCount}:`);
+    }
+    
+    return lastRunResult;
+  } catch (err) {
+    console.error('[Scheduler] Unhandled error during collection:', err.message);
+    lastRunResult = { success: false, error: err.message, run_count: runCount };
+    return lastRunResult;
+  } finally {
+    isRunning = false;
+  }
+}
+
+/**
+ * Start the scheduler (cron job + immediate first run).
+ */
+function startScheduler() {
+  console.log(`[Scheduler] Starting proxy data collector`);
+  const modeLabel = COLLECT_MODE === 'agent'
+    ? `SPECIFIC AGENT (${SPECIFIC_AGENT})`
+    : COLLECT_MODE === 'agents'
+    ? `SPECIFIC AGENTS (${SPECIFIC_AGENTS.join(', ')})`
+    : 'ALL AGENTS';
+  console.log(`[Scheduler] Mode     : ${modeLabel}`);
+  console.log(`[Scheduler] Schedule : ${CRON_SCHEDULE} (every 5 minutes by default)`);
+
+  // Validate cron expression
+  if (!cron.validate(CRON_SCHEDULE)) {
+    console.error(`[Scheduler] Invalid cron expression: "${CRON_SCHEDULE}". Using default.`);
+  }
+
+  // Start recurring cron job
+  cron.schedule(CRON_SCHEDULE, () => {
+    runCollection().catch(err => console.error('[Scheduler] Cron error:', err.message));
+  });
+
+  console.log('[Scheduler] Cron job registered. Starting initial collection...');
+  
+  // Initial run immediately on startup (async, do not block server start)
+  setTimeout(async () => {
+    // 1. Sync dictionary first
+    try {
+      await syncApplicationDictionary();
+    } catch (err) {
+      console.error('[Scheduler] Error syncing application dictionary:', err.message);
+    }
+    
+    // 2. Start normal telemetry collection
+    runCollection().catch(err => console.error('[Scheduler] Initial run error:', err.message));
+  }, 2000);
+}
+
+/**
+ * Get current scheduler status (for REST API endpoint).
+ */
+function getStatus() {
+  return {
+    is_running:     isRunning,
+    run_count:      runCount,
+    last_run_at:    lastRunAt?.toISOString() ?? null,
+    collect_mode:   COLLECT_MODE,
+    agent_uuid:     SPECIFIC_AGENT,
+    agent_uuids:    COLLECT_MODE === 'agents' ? SPECIFIC_AGENTS : [],
+    agent_delay_ms: AGENT_DELAY_MS,
+    cron_schedule:  CRON_SCHEDULE,
+    last_result:    lastRunResult,
+  };
+}
+
+module.exports = { startScheduler, runCollection, getStatus };