feat(source2): push all latest files - device labeling, help system, proxy docs, database isolation fix

- Added DOKUMENTASI-FILTER-PER-SITE.md (site isolation docs)
- Fixed start-with-env.js to force-load .env.production
- Fixed MONGODB_URI hostname from mongodb-netify to mongodb.prod.proit.id
- Updated .gitignore to exclude sensitive scripts and credential files
- Minor UI and labeling improvements

.gitignore

@@ -66,3 +66,19 @@ CLAUDE.md
 docs/
 plans/
 .env*
+
+# Local uploads
+backend/public/api/uploads/
+
+# Sensitive helper scripts (contain hardcoded SSH/API credentials - local use only)
+compare-netify-vs-dashboard.js
+ssh-read-source1-proxy.js
+check-frontend-uri-now.js
+verify-final.js
+check-frontend-uri.js
+ssh-check-logs.js
+ssh-*.js
+
+# Sensitive documentation (contains production API keys / credentials)
+BUKTI-AKSES-MONGODB.txt
+DOKUMENTASI-PROXY-NETIFY.md

.npmrc

@@ -0,0 +1 @@
+legacy-peer-deps=true

DOKUMENTASI-FILTER-PER-SITE.md

@@ -0,0 +1,252 @@
+# DETAIL TEKNIS: Cara Proxy Memfilter Data per Site (SIAB vs Office)
+
+Dokumen ini menjelaskan **secara kode** bagaimana data dipisahkan per site.
+Ada **3 lapis filter** yang bekerja dari Netify API sampai ke tampilan dashboard.
+
+---
+
+## LAPIS 1 — Saat Minta Data ke Netify API
+### File: `proxy/netifyClientCore.js`
+
+```
+NETIFY_SITE_UUIDS = "6681452d_....(SIAB), 1959bb55_....(Office)"
+                          |
+              proxy loop satu per satu:
+              ┌─────────────────────────┐
+              │  for SIAB UUID:         │
+              │    kirim request ke     │
+              │    Netify dengan header │
+              │    x-net-site: SIAB-UUID│
+              └─────────────────────────┘
+              ┌─────────────────────────┐
+              │  for Office UUID:       │
+              │    kirim request ke     │
+              │    Netify dengan header │
+              │    x-net-site: OFFICE-UUID│
+              └─────────────────────────┘
+```
+
+**KODE ASLI — cara header dikirim:**
+```javascript
+// proxy/netifyClientCore.js baris 14-18
+function getHeaders(siteUuid) {
+  const headers = {
+    'x-api-key': process.env.NETIFY_API_KEY,
+    'Accept': 'application/json'
+  };
+
+  if (siteUuid) headers['x-net-site'] = siteUuid;
+  //            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  // Ini yang memfilter data di sisi Netify!
+  // Netify API hanya kembalikan data untuk site ini saja.
+
+  return headers;
+}
+
+async function netifyFetch(endpoint, params = {}, agentUuid, siteUuid) {
+  const res = await axios.get(`${BASE_URL}${endpoint}`, {
+    headers: getHeaders(siteUuid),  // <--- siteUuid dikirim ke Netify
+    params,
+    timeout: 30000,
+  });
+}
+```
+
+**Artinya:** Netify API sendiri yang memfilter. Kalau kita kirim header
+`x-net-site: SIAB-UUID`, Netify HANYA kembalikan data milik SIAB.
+Kita tidak perlu filter manual — Netify sudah filter dari sumbernya.
+
+---
+
+## LAPIS 2 — Saat Simpan ke MongoDB
+### File: `proxy/collector.js` (loop utama)
+
+Setelah data dari Netify masuk, setiap dokumen diberi **stempel `site_uuid`**
+sebelum disimpan ke MongoDB.
+
+**KODE ASLI — loop per site di collector.js:**
+```javascript
+// proxy/collector.js baris 123-222
+
+// SITE_UUIDS diambil dari env:
+// NETIFY_SITE_UUIDS="6681452d_..., 1959bb55_..."
+const SITE_UUIDS = SITE_UUIDS_STR.split(','); // ["SIAB-UUID", "OFFICE-UUID"]
+
+for (const siteUuid of SITE_UUIDS) {
+//   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+//   Loop: pertama SIAB, lalu Office (satu per satu)
+
+  console.log(`Fetching agents for Site: ${siteUuid}`);
+  const agents = await netify.fetchAgents(siteUuid);
+  //                                      ^^^^^^^^^
+  //  fetchAgents pakai siteUuid → Netify hanya beri agent milik site ini
+
+  // --- PENTING: Anti-duplikat antar site ---
+  // Kadang Netify bisa kembalikan agent yang sama untuk 2 site.
+  // Di sini kita cegah agar 1 agent hanya masuk 1 site.
+  const agents = rawAgents.filter(a => {
+    if (processedAgentUuids.has(a.uuid)) {
+      console.log(`Skipping ${a.uuid} — already assigned to another site.`);
+      return false;  // lewati agent yang sudah diproses site lain
+    }
+    return true;
+  });
+  for (const agent of agents) processedAgentUuids.add(agent.uuid);
+
+  // Simpan agent ke MongoDB dengan site_uuid
+  await AgentRegistry.findOneAndUpdate(
+    { uuid: agent.uuid },
+    { $set: {
+        uuid: agent.uuid,
+        site_uuid: siteUuid,  // <--- stempel site di sini!
+        ...
+    }},
+    { upsert: true }
+  );
+
+  // Kumpulkan data untuk setiap agent di site ini
+  for (const agent of agents) {
+    await collectForAgent(agent.uuid, timestamp, siteUuid);
+    //                                           ^^^^^^^^^
+    //   siteUuid terus dibawa ke setiap fungsi collect
+  }
+}
+```
+
+**KODE ASLI — cara flows disimpan dengan site_uuid:**
+```javascript
+// proxy/collectorHelperDpi2.js baris 88-98
+
+const flowDocs = flows.map(f => ({
+  timestamp,
+  agent_uuid: agentUuid,   // siapa agent-nya
+  site_uuid:  SITE_UUID,   // <--- data ini milik site mana! (SIAB atau Office)
+  flow_id:    f.flow_id,
+  src_ip:     f.src_ip,
+  dst_ip:     f.dst_ip,
+  download:   f.download,
+  upload:     f.upload,
+  // ...
+}));
+
+// Upsert ke MongoDB (tidak duplikat berdasarkan flow_id + agent_uuid)
+await Flow.bulkWrite(flowDocs.map(f => ({
+  updateOne: {
+    filter: { flow_id: f.flow_id, agent_uuid: f.agent_uuid },
+    update: { $set: f },
+    upsert: true,
+  }
+})));
+```
+
+**Hasilnya di MongoDB — data terpisah per site:**
+```
+Collection: flows
+┌────────────────────┬────────────────────────────────────────────────────┬────────┬──────────┐
+│ flow_id            │ site_uuid                                          │ src_ip │ download │
+├────────────────────┼────────────────────────────────────────────────────┼────────┼──────────┤
+│ flow-001           │ 6681452d_9cae_4ff4_8ae8_0d504774265e  (SIAB)       │ 10.0.x │ 1234     │
+│ flow-002           │ 6681452d_9cae_4ff4_8ae8_0d504774265e  (SIAB)       │ 10.0.x │ 5678     │
+│ flow-003           │ 1959bb55_045b_47c7_bbdd_f33b7db197b9  (Office)     │ 192.168.x │ 9012  │
+│ flow-004           │ 1959bb55_045b_47c7_bbdd_f33b7db197b9  (Office)     │ 192.168.x │ 3456  │
+└────────────────────┴────────────────────────────────────────────────────┴────────┴──────────┘
+                       ^^^^^^^^^^ Field ini yang memisahkan data ^^^^^^^^^^
+```
+
+**Semua collection lain juga sama:**
+- `devices`   → tiap dokumen ada `site_uuid`
+- `threats`   → tiap dokumen ada `site_uuid`
+- `events`    → tiap dokumen ada `site_uuid`
+- `summaries` → tiap dokumen ada `site_uuid`
+- `telemetry` → tiap dokumen ada `site_uuid`
+
+---
+
+## LAPIS 3 — Saat Dashboard Baca dari MongoDB
+### File: `backend/routes/dashboard/flows.js` (contoh)
+
+Ketika user login sebagai admin SIAB dan buka halaman Flows,
+backend hanya query dokumen dengan `site_uuid` yang sesuai:
+
+```javascript
+// backend/routes/dashboard/flows.js (contoh query)
+const userSiteUuid = req.user.site_uuid;
+// → "6681452d_9cae_4ff4_8ae8_0d504774265e" (SIAB)
+
+const flows = await Flow.find({
+  site_uuid: userSiteUuid,  // <--- hanya ambil data site ini!
+  // ...filter waktu, pagination, dsb
+}).limit(50);
+```
+
+Admin Office login → `site_uuid = 1959bb55_...` → hanya lihat data Office.
+Admin SIAB login → `site_uuid = 6681452d_...` → hanya lihat data SIAB.
+Super Admin → bisa pilih site mana yang ingin dilihat.
+
+---
+
+## RINGKASAN — Alur Lengkap Filter Data
+
+```
+Netify API
+    |
+    |-- Lapis 1: Header x-net-site dikirim ke Netify
+    |   Netify hanya kirim data milik site tersebut
+    |
+    v
+Proxy Server (setiap 5 menit)
+    |
+    |-- Lapis 2: Setiap dokumen diberi stempel site_uuid
+    |   - SIAB data → { site_uuid: "6681452d_..." }
+    |   - Office data → { site_uuid: "1959bb55_..." }
+    |   - Anti-duplikat: 1 agent hanya masuk 1 site
+    |
+    v
+MongoDB (semua data tercampur tapi ter-tag per site)
+    |
+    |-- Lapis 3: Backend query MongoDB dengan filter site_uuid
+    |   - Admin SIAB login → WHERE site_uuid = SIAB-UUID
+    |   - Admin Office login → WHERE site_uuid = OFFICE-UUID
+    |
+    v
+Web Dashboard (tampil hanya data site yang sesuai)
+```
+
+---
+
+## Skenario Konkret
+
+**Skenario:** Network agent "F6-2V-DT-8A" ada di SIAB. Network agent "23-TE-6L-I2" ada di Office.
+
+### Langkah 1 — Proxy request ke Netify
+```
+[Iter 1] siteUuid = "6681452d..." (SIAB)
+  → GET /data/flows
+    Header: x-net-site: 6681452d...
+  → Netify kembalikan: flows dari F6-2V-DT-8A (agent SIAB)
+  → Simpan ke MongoDB: { site_uuid: "6681452d...", agent_uuid: "F6-2V-DT-8A", flow_id: ... }
+
+[Iter 2] siteUuid = "1959bb55..." (Office)
+  → GET /data/flows
+    Header: x-net-site: 1959bb55...
+  → Netify kembalikan: flows dari 23-TE-6L-I2 (agent Office)
+  → Simpan ke MongoDB: { site_uuid: "1959bb55...", agent_uuid: "23-TE-6L-I2", flow_id: ... }
+```
+
+### Langkah 2 — Dashboard tampilkan
+```
+User siab login:
+  req.user.site_uuid = "6681452d..."
+  DB query: Flow.find({ site_uuid: "6681452d..." })
+  Hasil: hanya flow dari F6-2V-DT-8A ✓
+
+User office login:
+  req.user.site_uuid = "1959bb55..."
+  DB query: Flow.find({ site_uuid: "1959bb55..." })
+  Hasil: hanya flow dari 23-TE-6L-I2 ✓
+```
+
+**Data tidak pernah tercampur** karena ada 3 lapis isolasi ini.
+
+---
+*Dokumentasi teknis Source 2 — 29 Juli 2026*

README.md

@@ -41,7 +41,7 @@ Produk BackOne oleh **PT. Data Bisnis Solusi** — Dashboard monitoring jaringan
 
 ## 📡 Proxy — 2 Mode Pengambilan Data
 
-Proxy server (port 4000) mendukung 3 mode yang dikontrol via environment variable:
+Proxy server (port 4000) mendukung 2 mode yang dikontrol via environment variable:
 
 ### Mode 1: Semua Network Agent (Admin BackOne)
 
@@ -62,26 +62,14 @@ PROXY_AGENT_UUID=2F-TF-1D-GK    # UUID Network Agent CPI Balaraja
 
 Proxy hanya mengambil data dari **satu Network Agent spesifik** (berdasarkan UUID). Data agent lain tidak pernah masuk ke database. Cocok untuk deployment di sisi client (Pihak A, B, C) agar mereka hanya punya data milik mereka sendiri.
 
-### Mode 3: Beberapa Agent Spesifik (Multi-Agent)
-
-```env
-# .env.local
-PROXY_COLLECT_MODE=agents
-PROXY_AGENT_UUIDS=UUID-AGENT-A,UUID-AGENT-B,UUID-AGENT-C   # comma-separated list
-PROXY_AGENT_DELAY_MS=5000                                   # delay antar agent (default: 5000ms)
-```
-
-Proxy mengambil data dari **beberapa Network Agent spesifik** (berdasarkan daftar UUID yang dipisahkan koma). Data agent di luar daftar tidak pernah masuk ke database. Delay antar agent dapat diatur dengan `PROXY_AGENT_DELAY_MS` untuk menghindari rate-limit.
-
 ### Contoh Multi-Tenant Deployment
 
-| Deployment | PROXY_COLLECT_MODE | PROXY_AGENT_UUID / PROXY_AGENT_UUIDS | Data yang disimpan |
+| Deployment | PROXY_COLLECT_MODE | PROXY_AGENT_UUID | Data yang disimpan |
 |---|---|---|---|
 | Kantor BackOne (Admin) | `all` | _(kosong)_ | Semua agent |
 | Pihak A | `agent` | `UUID-AGENT-A` | Hanya data Pihak A |
 | Pihak B | `agent` | `UUID-AGENT-B` | Hanya data Pihak B |
 | Pihak C | `agent` | `UUID-AGENT-C` | Hanya data Pihak C |
-| Multi-Client | `agents` | `UUID-A,UUID-B` | Data Pihak A dan B |
 
 ---
 
@@ -94,7 +82,6 @@ Proxy mengambil data dari **beberapa Network Agent spesifik** (berdasarkan dafta
 | GET | `/agents` | List semua agent UUID yang ada di MongoDB |
 | POST | `/collect/all` | Trigger manual — kumpulkan semua agent |
 | POST | `/collect/:uuid` | Trigger manual — kumpulkan agent spesifik |
-| POST | `/collect/agents` | Trigger manual — kumpulkan multiple agents (body: `{"uuids": [...], "delay_ms": 5000}`) |
 
 ---
 

SKILLS.md

@@ -26,9 +26,7 @@ contract, not just a task description.
 
 **Responsibilities**
 - Implement API/data-layer logic.
-- Wire the mock-vs-live routing required by the Demo/Live switch (`AGENTS.md` §5) and
+- Ensure all endpoints aggregate real-time data from MongoDB and Netify, avoiding any mock or simulated responses.
-  the Cloud/Local endpoint switch (`AGENTS.md` §6) — both must resolve through the
-  same contract so swapping either setting never changes calling code.
 - Keep business logic out of route handlers; route handlers stay thin.
 
 **When invoked**: any task touching data, APIs, or service integration.
@@ -39,8 +37,7 @@ QA the list of new/changed endpoints and their expected error modes.
 ## 3. Frontend Engineer
 
 **Responsibilities**
-- Implement UI for the task, including the Demo/Live and Cloud/Local switcher
+- Implement UI for the task.
-  controls where relevant.
 - Consume the Backend's contract rather than reaching around it.
 - Keep components small and composable, respecting the 256-LOC rule.
 

backend/Dockerfile.bun

@@ -0,0 +1,15 @@
+FROM oven/bun:1-alpine
+
+WORKDIR /app
+
+COPY backend/package*.json ./
+RUN bun install --production
+
+COPY backend/ .
+
+EXPOSE 3001
+
+HEALTHCHECK --interval=30s --timeout=10s --start-period=20s --retries=3 \
+  CMD bun -e "require('http').get('http://localhost:3001/api/health', r => r.statusCode === 200 ? process.exit(0) : process.exit(1)).on('error', () => process.exit(1))"
+
+CMD ["bun", "run", "server.js"]

backend/check_devices.js

@@ -0,0 +1,3 @@
+const db = require('better-sqlite3')('backend/netify_data.db');
+console.log('Devices:', db.prepare("SELECT * FROM devices WHERE ip_address = '192.168.9.2'").all());
+console.log('Discovery:', db.prepare("SELECT * FROM intel_device_discovery WHERE ip_address = '192.168.9.2'").all());

backend/check_events.js

@@ -0,0 +1,22 @@
+const mongoose = require('mongoose');
+require('dotenv').config({path: '../.env.local'});
+mongoose.connect(process.env.MONGODB_URI).then(async () => {
+  const db = mongoose.connection;
+  const highEvents = await db.collection('events').find({
+    $or: [
+      {severity: {$in: ['Critical', 'High']}}, 
+      {category_label: 'Cybersecurity'}
+    ]
+  }).toArray();
+  
+  if (highEvents.length > 0) {
+     console.log("High Events timestamps:");
+     highEvents.forEach(e => {
+        console.log("- event_at:", e.event_at, " | timestamp:", e.timestamp);
+     });
+  } else {
+     console.log("No high events found in array");
+  }
+
+  process.exit(0);
+}).catch(e => console.error(e));

backend/check_overview.js

@@ -0,0 +1,44 @@
+const mongoose = require('mongoose');
+
+mongoose.connect('mongodb://backone_user:SusuKudaLiar@103.80.237.29:27017/backone_dpi?authSource=backone_dpi')
+  .then(async () => {
+    const db = mongoose.connection.useDb('backone_dpi');
+    const yesterday = new Date(Date.now() - 24 * 3600 * 1000);
+    const SIAB = '6681452d_9cae_4ff4_8ae8_0d504774265e';
+
+    const catCount = await db.db.collection('appcategorystats').countDocuments({ site_uuid: SIAB, timestamp: { $gte: yesterday } });
+    const catSum = await db.db.collection('appcategorystats').aggregate([
+      { $match: { site_uuid: SIAB, timestamp: { $gte: yesterday } } },
+      { $group: { _id: null, dl: { $sum: '$download' }, ul: { $sum: '$upload' } } }
+    ]).toArray();
+
+    const sumCount = await db.db.collection('summaries').countDocuments({ site_uuid: SIAB, timestamp: { $gte: yesterday } });
+    const sumSum = await db.db.collection('summaries').aggregate([
+      { $match: { site_uuid: SIAB, timestamp: { $gte: yesterday } } },
+      { $group: { _id: null, dl: { $sum: '$bandwidth_down' }, ul: { $sum: '$bandwidth_up' } } }
+    ]).toArray();
+
+    const flowCount = await db.db.collection('flows').countDocuments({ site_uuid: SIAB, timestamp: { $gte: yesterday } });
+    const flowSum = await db.db.collection('flows').aggregate([
+      { $match: { site_uuid: SIAB, timestamp: { $gte: yesterday } } },
+      { $group: { _id: null, dl: { $sum: '$download' }, ul: { $sum: '$upload' } } }
+    ]).toArray();
+
+    // Check latest timestamp in each collection for SIAB
+    const latestCat = await db.db.collection('appcategorystats').findOne({ site_uuid: SIAB }, { sort: { timestamp: -1 } });
+    const latestFlow = await db.db.collection('flows').findOne({ site_uuid: SIAB }, { sort: { timestamp: -1 } });
+    const latestSum = await db.db.collection('summaries').findOne({ site_uuid: SIAB }, { sort: { timestamp: -1 } });
+
+    console.log('=== SIAB Site Data Check (Last 24h) ===');
+    console.log('AppCatStats (24h):', catCount, 'docs | Sum:', JSON.stringify(catSum[0]));
+    console.log('Summaries (24h)  :', sumCount, 'docs | Sum:', JSON.stringify(sumSum[0]));
+    console.log('Flows (24h)      :', flowCount, 'docs | Sum:', JSON.stringify(flowSum[0]));
+    console.log('');
+    console.log('=== Latest Timestamps ===');
+    console.log('Latest AppCat    :', latestCat?.timestamp);
+    console.log('Latest Flow      :', latestFlow?.timestamp);
+    console.log('Latest Summary   :', latestSum?.timestamp);
+
+    mongoose.disconnect();
+  })
+  .catch(e => { console.error('Error:', e.message); process.exit(1); });

backend/check_server.js

@@ -0,0 +1,31 @@
+const { Client } = require('ssh2');
+const conn = new Client();
+
+conn.on('ready', () => {
+  const cmd = [
+    'export PM2=/home/adminbackend/.npm-global/bin/pm2',
+    '$PM2 list',
+    'echo "=== MEMORY ==="',
+    'free -m',
+    'echo "=== DISK ==="',
+    'df -h /',
+    'echo "=== FRONTEND LOGS ==="',
+    '$PM2 logs backone-frontend --lines 20 --nostream 2>&1',
+    'echo "=== BACKEND LOGS ==="',
+    '$PM2 logs backone-backend --lines 10 --nostream 2>&1',
+  ].join(' && ');
+
+  conn.exec(cmd, (err, stream) => {
+    if (err) { console.error(err); conn.end(); return; }
+    stream.on('data', d => process.stdout.write(d.toString()));
+    stream.stderr.on('data', d => process.stderr.write(d.toString()));
+    stream.on('close', () => conn.end());
+  });
+}).connect({
+  host: '103.185.47.52',
+  port: 2222,
+  username: 'adminbackend',
+  password: 'htEo7x6LsBQiEHHH',
+});
+
+conn.on('error', e => console.error('SSH Error:', e.message));

backend/check_threats.js

@@ -0,0 +1,15 @@
+const mongoose = require('mongoose');
+require('dotenv').config({path: '../.env.local'});
+mongoose.connect(process.env.MONGODB_URI).then(async () => {
+  const db = mongoose.connection;
+  const threats = await db.collection('threats').countDocuments();
+  const events = await db.collection('events').countDocuments();
+  const highEvents = await db.collection('events').countDocuments({
+    $or: [
+      {severity: {$in: ['Critical', 'High']}}, 
+      {category_label: 'Cybersecurity'}
+    ]
+  });
+  console.log({threats, events, highEvents});
+  process.exit(0);
+}).catch(e => console.error(e));

backend/check_types.js

@@ -0,0 +1,10 @@
+const mongoose = require('mongoose');
+require('dotenv').config({path: '../.env.local'});
+mongoose.connect(process.env.MONGODB_URI).then(async () => {
+  const db = mongoose.connection;
+  const threats = await db.collection('threats').aggregate([{ $group: { _id: '$threat_type', count: { $sum: 1 } } }]).toArray();
+  console.log('Threat types:', threats);
+  const events = await db.collection('events').aggregate([{ $group: { _id: '$event_type', count: { $sum: 1 } } }]).toArray();
+  console.log('Event types:', events);
+  process.exit(0);
+});

backend/db/mongoose.js

@@ -5,7 +5,8 @@
 
 const mongoose = require('mongoose');
 const path = require('path');
-require('dotenv').config({ path: path.join(__dirname, '../../.env.local') });
+const envFile = process.env.NODE_ENV === 'production' ? '.env.production' : '.env.local';
+require('dotenv').config({ path: path.join(__dirname, '../../', envFile) });
 
 const MONGODB_URI = process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017/backone_dpi';
 
@@ -22,7 +23,7 @@ async function connectDB() {
         serverSelectionTimeoutMS: 10000,
         connectTimeoutMS: 10000,
       });
-      console.log('[MongoDB] ✓ Connected successfully');
+      console.log('[MongoDB] ✓ Connected successfully to', MONGODB_URI);
       const { logCapacityStats } = require('./capacityTracker');
       logCapacityStats('[MongoDB]').catch(err => console.warn('[MongoDB] Capacity log failed:', err.message));
       return;

backend/generate_export.js

@@ -1,400 +1,400 @@
-/**
+/**
- * generate_export.js
+ * generate_export.js
- * 
+ * 
- * Mengekspor SELURUH data dari semua tabel SQLite (database)
+ * Mengekspor SELURUH data dari semua tabel SQLite (database)
- * ke dalam file backone_data_export.txt
+ * ke dalam file backone_data_export.txt
- * 
+ * 
- * Format output:
+ * Format output:
- * - Header metadata (tanggal, versi, jumlah tabel)
+ * - Header metadata (tanggal, versi, jumlah tabel)
- * - Untuk setiap tabel: header section, row count, schema, dan semua data (JSON per baris)
+ * - Untuk setiap tabel: header section, row count, schema, dan semua data (JSON per baris)
- * - Footer summary
+ * - Footer summary
- */
+ */
-
+
-const fs   = require('fs');
+const fs   = require('fs');
-const path = require('path');
+const path = require('path');
-const db   = require('./database');
+const db   = require('./database');
-
+
-const OUTPUT_FILE = path.join(__dirname, '../backone_data_export.txt');
+const OUTPUT_FILE = path.join(__dirname, '../backone_data_export.txt');
-const d = db.getDB();
+const d = db.getDB();
-
+
-// ─── Helpers ────────────────────────────────────────────────────────────────
+// ─── Helpers ────────────────────────────────────────────────────────────────
-function fmtBytes(bytes) {
+function fmtBytes(bytes) {
-  if (!bytes || bytes === 0) return '0 B';
+  if (!bytes || bytes === 0) return '0 B';
-  const units = ['B', 'KB', 'MB', 'GB', 'TB'];
+  const units = ['B', 'KB', 'MB', 'GB', 'TB'];
-  let b = Math.abs(bytes);
+  let b = Math.abs(bytes);
-  let i = 0;
+  let i = 0;
-  while (b >= 1024 && i < units.length - 1) { b /= 1024; i++; }
+  while (b >= 1024 && i < units.length - 1) { b /= 1024; i++; }
-  return b.toFixed(2) + ' ' + units[i];
+  return b.toFixed(2) + ' ' + units[i];
-}
+}
-
+
-function fmtNum(n) {
+function fmtNum(n) {
-  if (n == null) return 'N/A';
+  if (n == null) return 'N/A';
-  return Number(n).toLocaleString('id-ID');
+  return Number(n).toLocaleString('id-ID');
-}
+}
-
+
-function separator(char = '═', len = 80) {
+function separator(char = '═', len = 80) {
-  return char.repeat(len);
+  return char.repeat(len);
-}
+}
-
+
-function sectionHeader(tableName, rowCount, description) {
+function sectionHeader(tableName, rowCount, description) {
-  return [
+  return [
-    '',
+    '',
-    separator('═'),
+    separator('═'),
-    `[TABLE: ${tableName}]`,
+    `[TABLE: ${tableName}]`,
-    `Row Count: ${fmtNum(rowCount)}`,
+    `Row Count: ${fmtNum(rowCount)}`,
-    description ? `Description: ${description}` : '',
+    description ? `Description: ${description}` : '',
-    separator('─'),
+    separator('─'),
-  ].filter(l => l !== '').join('\n');
+  ].filter(l => l !== '').join('\n');
-}
+}
-
+
-// ─── Table descriptions ──────────────────────────────────────────────────────
+// ─── Table descriptions ──────────────────────────────────────────────────────
-const TABLE_DESCRIPTIONS = {
+const TABLE_DESCRIPTIONS = {
-  bandwidth_apps       : 'Bandwidth per aplikasi (YouTube, Facebook, dll) dari BackOne DPI',
+  bandwidth_apps       : 'Bandwidth per aplikasi (YouTube, Facebook, dll) dari BackOne DPI',
-  bandwidth_timeline   : 'Timeline bandwidth per menit (download/upload historis)',
+  bandwidth_timeline   : 'Timeline bandwidth per menit (download/upload historis)',
-  bittorrent_info_hashes: 'Deteksi aktivitas BitTorrent berdasarkan info hash',
+  bittorrent_info_hashes: 'Deteksi aktivitas BitTorrent berdasarkan info hash',
-  countries            : 'Distribusi traffic berdasarkan negara tujuan',
+  countries            : 'Distribusi traffic berdasarkan negara tujuan',
-  devices              : 'Daftar perangkat (IP/MAC) beserta bandwidth & OS',
+  devices              : 'Daftar perangkat (IP/MAC) beserta bandwidth & OS',
-  dhcp_fingerprints    : 'Fingerprint DHCP untuk identifikasi tipe device',
+  dhcp_fingerprints    : 'Fingerprint DHCP untuk identifikasi tipe device',
-  discovered_os        : 'OS yang terdeteksi dari traffic scanning',
+  discovered_os        : 'OS yang terdeteksi dari traffic scanning',
-  dns_stats            : 'Query DNS teratas dan statistik resolusi domain',
+  dns_stats            : 'Query DNS teratas dan statistik resolusi domain',
-  events               : 'Event log dari BackOne agent (koneksi, peringatan, dll)',
+  events               : 'Event log dari BackOne agent (koneksi, peringatan, dll)',
-  flows                : 'Data aliran jaringan per-sesi (src IP, dst IP, aplikasi, domain, bytes)',
+  flows                : 'Data aliran jaringan per-sesi (src IP, dst IP, aplikasi, domain, bytes)',
-  flow_origins         : 'Asal flow: lokal (LAN) atau eksternal (WAN)',
+  flow_origins         : 'Asal flow: lokal (LAN) atau eksternal (WAN)',
-  flow_types           : 'Tipe flow: TCP, UDP, ICMP, dll',
+  flow_types           : 'Tipe flow: TCP, UDP, ICMP, dll',
-  http_user_agents     : 'HTTP User-Agent yang terdeteksi (browser, OS, framework)',
+  http_user_agents     : 'HTTP User-Agent yang terdeteksi (browser, OS, framework)',
-  intel_crypto_mining  : 'Deteksi aktivitas crypto mining (pool host, protokol)',
+  intel_crypto_mining  : 'Deteksi aktivitas crypto mining (pool host, protokol)',
-  intel_device_discovery: 'Penemuan perangkat baru di jaringan (tipe, OS, manufaktur)',
+  intel_device_discovery: 'Penemuan perangkat baru di jaringan (tipe, OS, manufaktur)',
-  intel_encryption_audit: 'Audit enkripsi traffic per perangkat (encrypted%, risk level)',
+  intel_encryption_audit: 'Audit enkripsi traffic per perangkat (encrypted%, risk level)',
-  intel_insecure_protocols: 'Protokol tidak aman yang terdeteksi (HTTP, Telnet, FTP, dll)',
+  intel_insecure_protocols: 'Protokol tidak aman yang terdeteksi (HTTP, Telnet, FTP, dll)',
-  intel_ip_reputation  : 'Reputasi IP eksternal (blacklist, threat score)',
+  intel_ip_reputation  : 'Reputasi IP eksternal (blacklist, threat score)',
-  intel_server_discovery: 'Server yang terdeteksi (HTTPS, SSH, HTTP, dll)',
+  intel_server_discovery: 'Server yang terdeteksi (HTTPS, SSH, HTTP, dll)',
-  intel_tor_detection  : 'Deteksi penggunaan jaringan Tor',
+  intel_tor_detection  : 'Deteksi penggunaan jaringan Tor',
-  intel_unencrypted_passwords: 'Deteksi pengiriman password dalam bentuk plaintext',
+  intel_unencrypted_passwords: 'Deteksi pengiriman password dalam bentuk plaintext',
-  intel_vpn_detection  : 'Deteksi penggunaan VPN (OpenVPN, WireGuard, dll)',
+  intel_vpn_detection  : 'Deteksi penggunaan VPN (OpenVPN, WireGuard, dll)',
-  interfaces           : 'Interface jaringan per agent (WAN/LAN, bandwidth)',
+  interfaces           : 'Interface jaringan per agent (WAN/LAN, bandwidth)',
-  ip_versions          : 'Distribusi traffic IPv4 vs IPv6',
+  ip_versions          : 'Distribusi traffic IPv4 vs IPv6',
-  mac_bandwidth        : 'Bandwidth per MAC address perangkat',
+  mac_bandwidth        : 'Bandwidth per MAC address perangkat',
-  mdns_hostnames       : 'mDNS hostname yang terdeteksi di jaringan lokal',
+  mdns_hostnames       : 'mDNS hostname yang terdeteksi di jaringan lokal',
-  netbios_hostnames    : 'NetBIOS hostname (nama komputer Windows)',
+  netbios_hostnames    : 'NetBIOS hostname (nama komputer Windows)',
-  protocols            : 'Distribusi protokol jaringan (port usage)',
+  protocols            : 'Distribusi protokol jaringan (port usage)',
-  quic_hostnames       : 'Hostname via QUIC/HTTP3 (Google, Cloudflare, dll)',
+  quic_hostnames       : 'Hostname via QUIC/HTTP3 (Google, Cloudflare, dll)',
-  regions              : 'Distribusi traffic berdasarkan region/kota tujuan',
+  regions              : 'Distribusi traffic berdasarkan region/kota tujuan',
-  remote_ips           : 'IP remote teratas yang diakses perangkat',
+  remote_ips           : 'IP remote teratas yang diakses perangkat',
-  sni_hostnames        : 'Server Name Indication dari koneksi TLS',
+  sni_hostnames        : 'Server Name Indication dari koneksi TLS',
-  ssh_versions         : 'Versi SSH yang terdeteksi di jaringan',
+  ssh_versions         : 'Versi SSH yang terdeteksi di jaringan',
-  ssl_server_cn        : 'Common Name sertifikat SSL server',
+  ssl_server_cn        : 'Common Name sertifikat SSL server',
-  threats              : 'Ancaman keamanan terdeteksi (threat alerts)',
+  threats              : 'Ancaman keamanan terdeteksi (threat alerts)',
-  tls_ciphers          : 'Cipher suite TLS yang digunakan',
+  tls_ciphers          : 'Cipher suite TLS yang digunakan',
-  tls_security         : 'Tingkat keamanan TLS (Modern, Compatible, Old)',
+  tls_security         : 'Tingkat keamanan TLS (Modern, Compatible, Old)',
-  tls_versions         : 'Versi TLS yang digunakan (1.0, 1.2, 1.3)',
+  tls_versions         : 'Versi TLS yang digunakan (1.0, 1.2, 1.3)',
-  vlans                : 'VLAN yang terdeteksi di jaringan',
+  vlans                : 'VLAN yang terdeteksi di jaringan',
-};
+};
-
+
-// ─── Main Export Logic ───────────────────────────────────────────────────────
+// ─── Main Export Logic ───────────────────────────────────────────────────────
-async function main() {
+async function main() {
-  console.log('🚀 Memulai export data...');
+  console.log('🚀 Memulai export data...');
-  
+  
-  const exportDate = new Date().toISOString();
+  const exportDate = new Date().toISOString();
-  const lines = [];
+  const lines = [];
-
+
-  // ── File Header ──────────────────────────────────────────────────────────
+  // ── File Header ──────────────────────────────────────────────────────────
-  lines.push(separator('═'));
+  lines.push(separator('═'));
-  lines.push('  BACKONE DATA EXPORT');
+  lines.push('  BACKONE DATA EXPORT');
-  lines.push('  Seluruh data hasil parsing dari BackOne API');
+  lines.push('  Seluruh data hasil parsing dari BackOne API');
-  lines.push(separator('─'));
+  lines.push(separator('─'));
-  lines.push(`  Export Date:   ${exportDate}`);
+  lines.push(`  Export Date:   ${exportDate}`);
-  lines.push(`  Generated by:  generate_export.js`);
+  lines.push(`  Generated by:  generate_export.js`);
-  lines.push(`  Source:        database (SQLite lokal)`);
+  lines.push(`  Source:        database (SQLite lokal)`);
-  lines.push(`  API Base:      BackOne API Service`);
+  lines.push(`  API Base:      BackOne API Service`);
-  lines.push(`  Format:        Per-tabel, data JSON satu record per baris (JSONL)`);
+  lines.push(`  Format:        Per-tabel, data JSON satu record per baris (JSONL)`);
-  lines.push(separator('─'));
+  lines.push(separator('─'));
-
+
-  // ── Get all tables ────────────────────────────────────────────────────────
+  // ── Get all tables ────────────────────────────────────────────────────────
-  const tables = d.prepare(
+  const tables = d.prepare(
-    "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
+    "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
-  ).all().map(r => r.name);
+  ).all().map(r => r.name);
-
+
-  lines.push(`  Total Tables:  ${tables.length}`);
+  lines.push(`  Total Tables:  ${tables.length}`);
-  lines.push(separator('═'));
+  lines.push(separator('═'));
-  lines.push('');
+  lines.push('');
-
+
-  // ── Table of Contents ─────────────────────────────────────────────────────
+  // ── Table of Contents ─────────────────────────────────────────────────────
-  lines.push('TABLE OF CONTENTS');
+  lines.push('TABLE OF CONTENTS');
-  lines.push(separator('─', 40));
+  lines.push(separator('─', 40));
-  let totalRows = 0;
+  let totalRows = 0;
-  const tableSummaries = [];
+  const tableSummaries = [];
-  for (const tableName of tables) {
+  for (const tableName of tables) {
-    const cnt = d.prepare(`SELECT COUNT(*) as c FROM ${tableName}`).get().c;
+    const cnt = d.prepare(`SELECT COUNT(*) as c FROM ${tableName}`).get().c;
-    totalRows += cnt;
+    totalRows += cnt;
-    const desc = TABLE_DESCRIPTIONS[tableName] || '-';
+    const desc = TABLE_DESCRIPTIONS[tableName] || '-';
-    lines.push(`  ${tableName.padEnd(35)} ${String(cnt).padStart(8)} rows`);
+    lines.push(`  ${tableName.padEnd(35)} ${String(cnt).padStart(8)} rows`);
-    tableSummaries.push({ name: tableName, count: cnt, description: desc });
+    tableSummaries.push({ name: tableName, count: cnt, description: desc });
-  }
+  }
-  lines.push(separator('─', 40));
+  lines.push(separator('─', 40));
-  lines.push(`  ${'TOTAL'.padEnd(35)} ${String(totalRows).padStart(8)} rows`);
+  lines.push(`  ${'TOTAL'.padEnd(35)} ${String(totalRows).padStart(8)} rows`);
-  lines.push('');
+  lines.push('');
-
+
-  // ── Per-Table Export ──────────────────────────────────────────────────────
+  // ── Per-Table Export ──────────────────────────────────────────────────────
-  for (const { name: tableName, count, description } of tableSummaries) {
+  for (const { name: tableName, count, description } of tableSummaries) {
-    console.log(`  📋 Exporting: ${tableName} (${fmtNum(count)} rows)...`);
+    console.log(`  📋 Exporting: ${tableName} (${fmtNum(count)} rows)...`);
-
+
-    // Section header
+    // Section header
-    lines.push(sectionHeader(tableName, count, description));
+    lines.push(sectionHeader(tableName, count, description));
-
+
-    // Schema
+    // Schema
-    const cols = d.prepare(`PRAGMA table_info(${tableName})`).all();
+    const cols = d.prepare(`PRAGMA table_info(${tableName})`).all();
-    lines.push('Schema:');
+    lines.push('Schema:');
-    cols.forEach(c => {
+    cols.forEach(c => {
-      lines.push(`  - ${c.name} [${c.type || 'TEXT'}]${c.notnull ? ' NOT NULL' : ''}${c.pk ? ' PRIMARY KEY' : ''}`);
+      lines.push(`  - ${c.name} [${c.type || 'TEXT'}]${c.notnull ? ' NOT NULL' : ''}${c.pk ? ' PRIMARY KEY' : ''}`);
-    });
+    });
-    lines.push('');
+    lines.push('');
-
+
-    // Statistics for numeric columns
+    // Statistics for numeric columns
-    const numericCols = cols.filter(c => 
+    const numericCols = cols.filter(c => 
-      ['INTEGER', 'REAL', 'NUMERIC'].includes((c.type || '').toUpperCase()) && 
+      ['INTEGER', 'REAL', 'NUMERIC'].includes((c.type || '').toUpperCase()) && 
-      !['id'].includes(c.name.toLowerCase())
+      !['id'].includes(c.name.toLowerCase())
-    );
+    );
-    
+    
-    if (count > 0 && numericCols.length > 0) {
+    if (count > 0 && numericCols.length > 0) {
-      lines.push('Statistics:');
+      lines.push('Statistics:');
-      for (const col of numericCols.slice(0, 5)) { // max 5 numeric cols
+      for (const col of numericCols.slice(0, 5)) { // max 5 numeric cols
-        try {
+        try {
-          const stat = d.prepare(`
+          const stat = d.prepare(`
-            SELECT MIN(${col.name}) as min, MAX(${col.name}) as max, 
+            SELECT MIN(${col.name}) as min, MAX(${col.name}) as max, 
-                   AVG(${col.name}) as avg, SUM(${col.name}) as total
+                   AVG(${col.name}) as avg, SUM(${col.name}) as total
-            FROM ${tableName}
+            FROM ${tableName}
-          `).get();
+          `).get();
-          if (stat && stat.max !== null) {
+          if (stat && stat.max !== null) {
-            lines.push(`  ${col.name}: min=${fmtNum(stat.min)} max=${fmtNum(stat.max)} avg=${Number(stat.avg || 0).toFixed(2)} total=${fmtNum(stat.total)}`);
+            lines.push(`  ${col.name}: min=${fmtNum(stat.min)} max=${fmtNum(stat.max)} avg=${Number(stat.avg || 0).toFixed(2)} total=${fmtNum(stat.total)}`);
-          }
+          }
-        } catch(e) { /* skip */ }
+        } catch(e) { /* skip */ }
-      }
+      }
-      lines.push('');
+      lines.push('');
-    }
+    }
-
+
-    // Data rows (ALL rows)
+    // Data rows (ALL rows)
-    if (count === 0) {
+    if (count === 0) {
-      lines.push('(No data)');
+      lines.push('(No data)');
-    } else {
+    } else {
-      lines.push(`Data (${fmtNum(count)} records):`);
+      lines.push(`Data (${fmtNum(count)} records):`);
-      const rows = d.prepare(`SELECT * FROM ${tableName}`).all();
+      const rows = d.prepare(`SELECT * FROM ${tableName}`).all();
-      for (const row of rows) {
+      for (const row of rows) {
-        lines.push(JSON.stringify(row));
+        lines.push(JSON.stringify(row));
-      }
+      }
-    }
+    }
-    lines.push('');
+    lines.push('');
-  }
+  }
-
+
-  // ── Agent-specific sections (derived from flows) ───────────────────────────
+  // ── Agent-specific sections (derived from flows) ───────────────────────────
-  lines.push('');
+  lines.push('');
-  lines.push(separator('═'));
+  lines.push(separator('═'));
-  lines.push('[DERIVED: AGENT ANALYSIS]');
+  lines.push('[DERIVED: AGENT ANALYSIS]');
-  lines.push('Description: Analisis traffic per agent berdasarkan MAC address dari flows table');
+  lines.push('Description: Analisis traffic per agent berdasarkan MAC address dari flows table');
-  lines.push(separator('─'));
+  lines.push(separator('─'));
-
+
-  const AGENT_MAC_MAP = {
+  const AGENT_MAC_MAP = {
-    '2F-TF-1D-GK': { label: 'JRP Cibubur',  macs: ['60:be:b4:1f:05:96'] },
+    '2F-TF-1D-GK': { label: 'JRP Cibubur',  macs: ['60:be:b4:1f:05:96'] },
-    '8A-V3-PB-85': { label: 'IFG LT.18',    macs: ['04:f4:1c:ce:c2:e6'] },
+    '8A-V3-PB-85': { label: 'IFG LT.18',    macs: ['04:f4:1c:ce:c2:e6'] },
-    'F6-2V-DT-8A': { label: 'CPI Balaraja', macs: ['2c:7b:a0:d8:86:91', '16:11:ac:73:34:1d', 'bc:45:5b:ca:d5:be', 'de:ed:cc:57:58:34', 'f4:6d:3f:ef:01:a0', '60:be:b4:29:d3:36'] },
+    'F6-2V-DT-8A': { label: 'CPI Balaraja', macs: ['2c:7b:a0:d8:86:91', '16:11:ac:73:34:1d', 'bc:45:5b:ca:d5:be', 'de:ed:cc:57:58:34', 'f4:6d:3f:ef:01:a0', '60:be:b4:29:d3:36'] },
-  };
+  };
-
+
-  for (const [uuid, agent] of Object.entries(AGENT_MAC_MAP)) {
+  for (const [uuid, agent] of Object.entries(AGENT_MAC_MAP)) {
-    lines.push('');
+    lines.push('');
-    lines.push(`Agent: ${agent.label} (${uuid})`);
+    lines.push(`Agent: ${agent.label} (${uuid})`);
-    lines.push(`MACs:  ${agent.macs.join(', ')}`);
+    lines.push(`MACs:  ${agent.macs.join(', ')}`);
-    lines.push(separator('─', 40));
+    lines.push(separator('─', 40));
-
+
-    const ph = agent.macs.map(() => '?').join(',');
+    const ph = agent.macs.map(() => '?').join(',');
-
+
-    // Summary
+    // Summary
-    const sumRow = d.prepare(`
+    const sumRow = d.prepare(`
-      SELECT COUNT(DISTINCT src_ip) AS device_count, COUNT(*) AS flow_count,
+      SELECT COUNT(DISTINCT src_ip) AS device_count, COUNT(*) AS flow_count,
-             SUM(bytes_download) AS total_dl, SUM(bytes_upload) AS total_ul
+             SUM(bytes_download) AS total_dl, SUM(bytes_upload) AS total_ul
-      FROM flows WHERE src_mac IN (${ph})
+      FROM flows WHERE src_mac IN (${ph})
-    `).get(...agent.macs);
+    `).get(...agent.macs);
-
+
-    lines.push(`  Devices:        ${fmtNum(sumRow.device_count)}`);
+    lines.push(`  Devices:        ${fmtNum(sumRow.device_count)}`);
-    lines.push(`  Total Flows:    ${fmtNum(sumRow.flow_count)}`);
+    lines.push(`  Total Flows:    ${fmtNum(sumRow.flow_count)}`);
-    lines.push(`  Total Download: ${fmtBytes(sumRow.total_dl)}`);
+    lines.push(`  Total Download: ${fmtBytes(sumRow.total_dl)}`);
-    lines.push(`  Total Upload:   ${fmtBytes(sumRow.total_ul)}`);
+    lines.push(`  Total Upload:   ${fmtBytes(sumRow.total_ul)}`);
-
+
-    // Top apps
+    // Top apps
-    const apps = d.prepare(`
+    const apps = d.prepare(`
-      SELECT app_label, SUM(bytes_download) AS dl, SUM(bytes_upload) AS ul, COUNT(*) AS cnt
+      SELECT app_label, SUM(bytes_download) AS dl, SUM(bytes_upload) AS ul, COUNT(*) AS cnt
-      FROM flows WHERE src_mac IN (${ph}) AND app_label IS NOT NULL
+      FROM flows WHERE src_mac IN (${ph}) AND app_label IS NOT NULL
-      GROUP BY app_label ORDER BY dl DESC LIMIT 10
+      GROUP BY app_label ORDER BY dl DESC LIMIT 10
-    `).all(...agent.macs);
+    `).all(...agent.macs);
-
+
-    lines.push(`  Top Applications:`);
+    lines.push(`  Top Applications:`);
-    apps.forEach((a, i) => {
+    apps.forEach((a, i) => {
-      lines.push(`    ${String(i+1).padStart(2)}. ${(a.app_label||'?').padEnd(30)} DL:${fmtBytes(a.dl).padStart(12)}  UL:${fmtBytes(a.ul).padStart(12)}  Flows:${a.cnt}`);
+      lines.push(`    ${String(i+1).padStart(2)}. ${(a.app_label||'?').padEnd(30)} DL:${fmtBytes(a.dl).padStart(12)}  UL:${fmtBytes(a.ul).padStart(12)}  Flows:${a.cnt}`);
-    });
+    });
-
+
-    // Top devices
+    // Top devices
-    const devs = d.prepare(`
+    const devs = d.prepare(`
-      SELECT src_ip, SUM(bytes_download) AS dl, MAX(last_seen) AS last
+      SELECT src_ip, SUM(bytes_download) AS dl, MAX(last_seen) AS last
-      FROM flows WHERE src_mac IN (${ph})
+      FROM flows WHERE src_mac IN (${ph})
-      GROUP BY src_ip ORDER BY dl DESC LIMIT 10
+      GROUP BY src_ip ORDER BY dl DESC LIMIT 10
-    `).all(...agent.macs);
+    `).all(...agent.macs);
-
+
-    lines.push(`  Top Devices:`);
+    lines.push(`  Top Devices:`);
-    devs.forEach((d2, i) => {
+    devs.forEach((d2, i) => {
-      lines.push(`    ${String(i+1).padStart(2)}. ${(d2.src_ip||'?').padEnd(20)} DL:${fmtBytes(d2.dl).padStart(12)}  Last:${d2.last||'-'}`);
+      lines.push(`    ${String(i+1).padStart(2)}. ${(d2.src_ip||'?').padEnd(20)} DL:${fmtBytes(d2.dl).padStart(12)}  Last:${d2.last||'-'}`);
-    });
+    });
-  }
+  }
-
+
-  // ── Bandwidth Apps Summary ─────────────────────────────────────────────────
+  // ── Bandwidth Apps Summary ─────────────────────────────────────────────────
-  lines.push('');
+  lines.push('');
-  lines.push(separator('═'));
+  lines.push(separator('═'));
-  lines.push('[DERIVED: BANDWIDTH APPS LATEST SNAPSHOT]');
+  lines.push('[DERIVED: BANDWIDTH APPS LATEST SNAPSHOT]');
-  lines.push('Description: Snapshot terakhir bandwidth per aplikasi (nilai aktual, bukan akumulasi)');
+  lines.push('Description: Snapshot terakhir bandwidth per aplikasi (nilai aktual, bukan akumulasi)');
-  lines.push(separator('─'));
+  lines.push(separator('─'));
-
+
-  const latestBwSnap = d.prepare('SELECT MAX(fetched_at) as t FROM bandwidth_apps').get()?.t;
+  const latestBwSnap = d.prepare('SELECT MAX(fetched_at) as t FROM bandwidth_apps').get()?.t;
-  if (latestBwSnap) {
+  if (latestBwSnap) {
-    lines.push(`Latest Snapshot: ${latestBwSnap}`);
+    lines.push(`Latest Snapshot: ${latestBwSnap}`);
-    const bwApps = d.prepare('SELECT app_label, category, download, upload, total, flow_count FROM bandwidth_apps WHERE fetched_at = ? ORDER BY download DESC').all(latestBwSnap);
+    const bwApps = d.prepare('SELECT app_label, category, download, upload, total, flow_count FROM bandwidth_apps WHERE fetched_at = ? ORDER BY download DESC').all(latestBwSnap);
-    lines.push(`Total Apps: ${bwApps.length}`);
+    lines.push(`Total Apps: ${bwApps.length}`);
-    lines.push('');
+    lines.push('');
-    bwApps.forEach((a, i) => {
+    bwApps.forEach((a, i) => {
-      lines.push(`  ${String(i+1).padStart(3)}. ${(a.app_label||'?').padEnd(30)} [${(a.category||'?').padEnd(20)}] DL:${fmtBytes(a.download).padStart(12)}  UL:${fmtBytes(a.upload).padStart(12)}  Flows:${fmtNum(a.flow_count)}`);
+      lines.push(`  ${String(i+1).padStart(3)}. ${(a.app_label||'?').padEnd(30)} [${(a.category||'?').padEnd(20)}] DL:${fmtBytes(a.download).padStart(12)}  UL:${fmtBytes(a.upload).padStart(12)}  Flows:${fmtNum(a.flow_count)}`);
-    });
+    });
-  }
+  }
-
+
-  // ── Encryption Audit Summary ───────────────────────────────────────────────
+  // ── Encryption Audit Summary ───────────────────────────────────────────────
-  lines.push('');
+  lines.push('');
-  lines.push(separator('═'));
+  lines.push(separator('═'));
-  lines.push('[DERIVED: ENCRYPTION RISK SUMMARY]');
+  lines.push('[DERIVED: ENCRYPTION RISK SUMMARY]');
-  lines.push('Description: Distribusi risk level enkripsi per perangkat (snapshot terbaru)');
+  lines.push('Description: Distribusi risk level enkripsi per perangkat (snapshot terbaru)');
-  lines.push(separator('─'));
+  lines.push(separator('─'));
-
+
-  const latestEncSnap = d.prepare('SELECT MAX(fetched_at) as t FROM intel_encryption_audit').get()?.t;
+  const latestEncSnap = d.prepare('SELECT MAX(fetched_at) as t FROM intel_encryption_audit').get()?.t;
-  if (latestEncSnap) {
+  if (latestEncSnap) {
-    const riskDist = d.prepare(`
+    const riskDist = d.prepare(`
-      SELECT risk_level, COUNT(*) as cnt, AVG(encrypted_pct) as avg_enc
+      SELECT risk_level, COUNT(*) as cnt, AVG(encrypted_pct) as avg_enc
-      FROM intel_encryption_audit WHERE fetched_at = ?
+      FROM intel_encryption_audit WHERE fetched_at = ?
-      GROUP BY risk_level ORDER BY cnt DESC
+      GROUP BY risk_level ORDER BY cnt DESC
-    `).all(latestEncSnap);
+    `).all(latestEncSnap);
-
+
-    lines.push(`Latest Snapshot: ${latestEncSnap}`);
+    lines.push(`Latest Snapshot: ${latestEncSnap}`);
-    lines.push('Risk Distribution:');
+    lines.push('Risk Distribution:');
-    riskDist.forEach(r => {
+    riskDist.forEach(r => {
-      lines.push(`  ${(r.risk_level||'Unknown').padEnd(15)} ${String(r.cnt).padStart(5)} devices   avg encrypted: ${Number(r.avg_enc||0).toFixed(1)}%`);
+      lines.push(`  ${(r.risk_level||'Unknown').padEnd(15)} ${String(r.cnt).padStart(5)} devices   avg encrypted: ${Number(r.avg_enc||0).toFixed(1)}%`);
-    });
+    });
-
+
-    // Highest risk devices
+    // Highest risk devices
-    lines.push('');
+    lines.push('');
-    lines.push('Critical Risk Devices (0% encrypted):');
+    lines.push('Critical Risk Devices (0% encrypted):');
-    const critDevs = d.prepare(`
+    const critDevs = d.prepare(`
-      SELECT ip_address, mac_address, device_label, encrypted_pct, total
+      SELECT ip_address, mac_address, device_label, encrypted_pct, total
-      FROM intel_encryption_audit WHERE fetched_at = ? AND risk_level = 'Critical'
+      FROM intel_encryption_audit WHERE fetched_at = ? AND risk_level = 'Critical'
-      ORDER BY total DESC LIMIT 20
+      ORDER BY total DESC LIMIT 20
-    `).all(latestEncSnap);
+    `).all(latestEncSnap);
-    critDevs.forEach(r => {
+    critDevs.forEach(r => {
-      lines.push(JSON.stringify(r));
+      lines.push(JSON.stringify(r));
-    });
+    });
-  }
+  }
-
+
-  // ── DNS Top Domains ────────────────────────────────────────────────────────
+  // ── DNS Top Domains ────────────────────────────────────────────────────────
-  lines.push('');
+  lines.push('');
-  lines.push(separator('═'));
+  lines.push(separator('═'));
-  lines.push('[DERIVED: TOP DNS DOMAINS]');
+  lines.push('[DERIVED: TOP DNS DOMAINS]');
-  lines.push('Description: Domain paling sering diquery dari DNS stats');
+  lines.push('Description: Domain paling sering diquery dari DNS stats');
-  lines.push(separator('─'));
+  lines.push(separator('─'));
-
+
-  const latestDnsSnap = d.prepare('SELECT MAX(fetched_at) as t FROM dns_queries').get()?.t;
+  const latestDnsSnap = d.prepare('SELECT MAX(fetched_at) as t FROM dns_queries').get()?.t;
-  if (latestDnsSnap) {
+  if (latestDnsSnap) {
-    const dnsRows = d.prepare('SELECT * FROM dns_queries WHERE fetched_at = ? ORDER BY query_count DESC LIMIT 30').all(latestDnsSnap);
+    const dnsRows = d.prepare('SELECT * FROM dns_queries WHERE fetched_at = ? ORDER BY query_count DESC LIMIT 30').all(latestDnsSnap);
-
+
-    lines.push(`Latest Snapshot: ${latestDnsSnap}`);
+    lines.push(`Latest Snapshot: ${latestDnsSnap}`);
-    dnsRows.forEach(r => lines.push(JSON.stringify(r)));
+    dnsRows.forEach(r => lines.push(JSON.stringify(r)));
-  }
+  }
-
+
-  // ── IP Reputation Blacklisted ─────────────────────────────────────────────
+  // ── IP Reputation Blacklisted ─────────────────────────────────────────────
-  lines.push('');
+  lines.push('');
-  lines.push(separator('═'));
+  lines.push(separator('═'));
-  lines.push('[DERIVED: BLACKLISTED IP ADDRESSES]');
+  lines.push('[DERIVED: BLACKLISTED IP ADDRESSES]');
-  lines.push('Description: IP address yang terdeteksi blacklisted (dari intel_ip_reputation)');
+  lines.push('Description: IP address yang terdeteksi blacklisted (dari intel_ip_reputation)');
-  lines.push(separator('─'));
+  lines.push(separator('─'));
-
+
-  const latestRepSnap = d.prepare('SELECT MAX(fetched_at) as t FROM intel_ip_reputation').get()?.t;
+  const latestRepSnap = d.prepare('SELECT MAX(fetched_at) as t FROM intel_ip_reputation').get()?.t;
-  if (latestRepSnap) {
+  if (latestRepSnap) {
-    const blacklisted = d.prepare('SELECT * FROM intel_ip_reputation WHERE fetched_at = ? AND blacklisted = 1').all(latestRepSnap);
+    const blacklisted = d.prepare('SELECT * FROM intel_ip_reputation WHERE fetched_at = ? AND blacklisted = 1').all(latestRepSnap);
-    lines.push(`Latest Snapshot: ${latestRepSnap}`);
+    lines.push(`Latest Snapshot: ${latestRepSnap}`);
-    lines.push(`Blacklisted count: ${blacklisted.length}`);
+    lines.push(`Blacklisted count: ${blacklisted.length}`);
-    blacklisted.forEach(r => lines.push(JSON.stringify(r)));
+    blacklisted.forEach(r => lines.push(JSON.stringify(r)));
-  }
+  }
-
+
-  // ── Flows: Active Sessions Summary ────────────────────────────────────────
+  // ── Flows: Active Sessions Summary ────────────────────────────────────────
-  lines.push('');
+  lines.push('');
-  lines.push(separator('═'));
+  lines.push(separator('═'));
-  lines.push('[DERIVED: ACTIVE FLOWS SUMMARY]');
+  lines.push('[DERIVED: ACTIVE FLOWS SUMMARY]');
-  lines.push('Description: Ringkasan aliran jaringan aktif (50 terbaru per download)');
+  lines.push('Description: Ringkasan aliran jaringan aktif (50 terbaru per download)');
-  lines.push(separator('─'));
+  lines.push(separator('─'));
-
+
-  const flowSummary = d.prepare(`
+  const flowSummary = d.prepare(`
-    SELECT COUNT(*) as total_flows,
+    SELECT COUNT(*) as total_flows,
-           COUNT(DISTINCT src_ip) as unique_src_ips,
+           COUNT(DISTINCT src_ip) as unique_src_ips,
-           COUNT(DISTINCT dst_ip) as unique_dst_ips,
+           COUNT(DISTINCT dst_ip) as unique_dst_ips,
-           COUNT(DISTINCT src_mac) as unique_macs,
+           COUNT(DISTINCT src_mac) as unique_macs,
-           SUM(bytes_download) as total_dl,
+           SUM(bytes_download) as total_dl,
-           SUM(bytes_upload) as total_ul,
+           SUM(bytes_upload) as total_ul,
-           MIN(first_seen) as earliest,
+           MIN(first_seen) as earliest,
-           MAX(last_seen) as latest
+           MAX(last_seen) as latest
-    FROM flows
+    FROM flows
-  `).get();
+  `).get();
-
+
-  lines.push(`Total Flows in DB:    ${fmtNum(flowSummary.total_flows)}`);
+  lines.push(`Total Flows in DB:    ${fmtNum(flowSummary.total_flows)}`);
-  lines.push(`Unique Source IPs:    ${fmtNum(flowSummary.unique_src_ips)}`);
+  lines.push(`Unique Source IPs:    ${fmtNum(flowSummary.unique_src_ips)}`);
-  lines.push(`Unique Dest IPs:      ${fmtNum(flowSummary.unique_dst_ips)}`);
+  lines.push(`Unique Dest IPs:      ${fmtNum(flowSummary.unique_dst_ips)}`);
-  lines.push(`Unique MAC Addresses: ${fmtNum(flowSummary.unique_macs)}`);
+  lines.push(`Unique MAC Addresses: ${fmtNum(flowSummary.unique_macs)}`);
-  lines.push(`Total Download:       ${fmtBytes(flowSummary.total_dl)}`);
+  lines.push(`Total Download:       ${fmtBytes(flowSummary.total_dl)}`);
-  lines.push(`Total Upload:         ${fmtBytes(flowSummary.total_ul)}`);
+  lines.push(`Total Upload:         ${fmtBytes(flowSummary.total_ul)}`);
-  lines.push(`Data from:            ${flowSummary.earliest}`);
+  lines.push(`Data from:            ${flowSummary.earliest}`);
-  lines.push(`Data to:              ${flowSummary.latest}`);
+  lines.push(`Data to:              ${flowSummary.latest}`);
-  lines.push('');
+  lines.push('');
-
+
-  // Top 50 flows by download
+  // Top 50 flows by download
-  lines.push('Top 50 Flows by Download:');
+  lines.push('Top 50 Flows by Download:');
-  const topFlows = d.prepare('SELECT * FROM flows ORDER BY bytes_download DESC LIMIT 50').all();
+  const topFlows = d.prepare('SELECT * FROM flows ORDER BY bytes_download DESC LIMIT 50').all();
-  topFlows.forEach(r => lines.push(JSON.stringify(r)));
+  topFlows.forEach(r => lines.push(JSON.stringify(r)));
-
+
-  // ── Unencrypted Password Events ───────────────────────────────────────────
+  // ── Unencrypted Password Events ───────────────────────────────────────────
-  lines.push('');
+  lines.push('');
-  lines.push(separator('═'));
+  lines.push(separator('═'));
-  lines.push('[DERIVED: UNENCRYPTED PASSWORD DETECTIONS]');
+  lines.push('[DERIVED: UNENCRYPTED PASSWORD DETECTIONS]');
-  lines.push('Description: Kejadian pengiriman credential dalam plaintext (HIGH severity)');
+  lines.push('Description: Kejadian pengiriman credential dalam plaintext (HIGH severity)');
-  lines.push(separator('─'));
+  lines.push(separator('─'));
-
+
-  const unencPwdHigh = d.prepare(`
+  const unencPwdHigh = d.prepare(`
-    SELECT ip_address, mac_address, dst_ip, dst_port, protocol, username, download, upload, severity, detected_at
+    SELECT ip_address, mac_address, dst_ip, dst_port, protocol, username, download, upload, severity, detected_at
-    FROM intel_unencrypted_passwords ORDER BY detected_at DESC
+    FROM intel_unencrypted_passwords ORDER BY detected_at DESC
-  `).all();
+  `).all();
-  lines.push(`Total detections: ${unencPwdHigh.length}`);
+  lines.push(`Total detections: ${unencPwdHigh.length}`);
-  unencPwdHigh.forEach(r => lines.push(JSON.stringify(r)));
+  unencPwdHigh.forEach(r => lines.push(JSON.stringify(r)));
-
+
-  // ── Footer ────────────────────────────────────────────────────────────────
+  // ── Footer ────────────────────────────────────────────────────────────────
-  lines.push('');
+  lines.push('');
-  lines.push(separator('═'));
+  lines.push(separator('═'));
-  lines.push('  END OF EXPORT');
+  lines.push('  END OF EXPORT');
-  lines.push(`  Generated at: ${new Date().toISOString()}`);
+  lines.push(`  Generated at: ${new Date().toISOString()}`);
-  lines.push(`  Total lines:  ${lines.length + 3}`);
+  lines.push(`  Total lines:  ${lines.length + 3}`);
-  lines.push(separator('═'));
+  lines.push(separator('═'));
-
+
-  // Write to file
+  // Write to file
-  const output = lines.join('\n');
+  const output = lines.join('\n');
-  fs.writeFileSync(OUTPUT_FILE, output, 'utf-8');
+  fs.writeFileSync(OUTPUT_FILE, output, 'utf-8');
-
+
-  const stats = fs.statSync(OUTPUT_FILE);
+  const stats = fs.statSync(OUTPUT_FILE);
-  console.log(`\n✅ Export selesai!`);
+  console.log(`\n✅ Export selesai!`);
-  console.log(`   File: ${OUTPUT_FILE}`);
+  console.log(`   File: ${OUTPUT_FILE}`);
-  console.log(`   Size: ${fmtBytes(stats.size)}`);
+  console.log(`   Size: ${fmtBytes(stats.size)}`);
-  console.log(`   Lines: ${fmtNum(lines.length)}`);
+  console.log(`   Lines: ${fmtNum(lines.length)}`);
-  console.log(`   Tables: ${tables.length}`);
+  console.log(`   Tables: ${tables.length}`);
-  console.log(`   Total Rows: ${fmtNum(totalRows)}`);
+  console.log(`   Total Rows: ${fmtNum(totalRows)}`);
-}
+}
-
+
-main().catch(e => {
+main().catch(e => {
-  console.error('❌ Export FAILED:', e);
+  console.error('❌ Export FAILED:', e);
-  process.exit(1);
+  process.exit(1);
-});
+});

backend/middleware/auth.js

@@ -1,5 +1,6 @@
 const jwt = require('jsonwebtoken');
 const User = require('../models/User');
+const Session = require('../models/Session');
 const { Summary } = require('../models/Schemas');
 
 const JWT_SECRET = process.env.JWT_SECRET || 'super-secret-backone-key';
@@ -11,19 +12,46 @@ async function requireAuth(req, res, next) {
   try {
     req.user = jwt.verify(token, JWT_SECRET);
 
+    // Verify session status in MongoDB
+    if (req.user.session_id) {
+      const activeSession = await Session.findById(req.user.session_id);
+      if (!activeSession) {
+        res.clearCookie('token');
+        return res.status(401).json({ error: 'Sesi login telah dinonaktifkan atau kedaluwarsa.' });
+      }
+      // Update last active
+      activeSession.last_active = new Date();
+      await activeSession.save();
+    }
+
     // ── VIEW-AS MODE ──────────────────────────────────────────────────────────
     const viewAsHeader = req.headers['x-view-as-agent'];
-    if (viewAsHeader && (req.user.role === 'SUPER_ADMIN' || req.user.role === 'TENANT_ADMIN')) {
+    const isAllowedViewAs = req.user.role === 'SUPER_ADMIN' || 
+      req.user.role === 'TENANT_ADMIN' || 
+      req.user.role === 'COMPANY_ADMIN' || 
+      req.user.role === 'COMPANY_OPERATOR';
+
+    if (viewAsHeader && isAllowedViewAs) {
       try {
         const viewDecoded = jwt.verify(viewAsHeader, JWT_SECRET);
         if (viewDecoded.type === 'view-as' && viewDecoded.adminId === req.user.id && viewDecoded.viewAs) {
-          const targetAgentUser = await User.findOne({ agent_uuid: viewDecoded.viewAs, role: 'AGENT_VIEWER' }).lean();
+          const targetAgent = viewDecoded.viewAs;
+
+          // Validation: COMPANY_ADMIN and COMPANY_OPERATOR can only view-as their assigned agents
+          if (['COMPANY_ADMIN', 'COMPANY_OPERATOR'].includes(req.user.role)) {
+            const hasAccess = req.user.agent_uuids && req.user.agent_uuids.includes(targetAgent);
+            if (!hasAccess) {
+              throw new Error('Unauthorized view-as agent access');
+            }
+          }
+
+          const targetAgentUser = await User.findOne({ agent_uuid: targetAgent, role: 'AGENT_VIEWER' }).lean();
           
           let targetSiteUuid = req.user.site_uuid;
           if (targetAgentUser && targetAgentUser.site_uuid) {
             targetSiteUuid = targetAgentUser.site_uuid;
           } else {
-            const summaryDoc = await Summary.findOne({ agent_uuid: viewDecoded.viewAs }).lean();
+            const summaryDoc = await Summary.findOne({ agent_uuid: targetAgent }).lean();
             if (summaryDoc && summaryDoc.site_uuid) {
               targetSiteUuid = summaryDoc.site_uuid;
             }
@@ -32,7 +60,7 @@ async function requireAuth(req, res, next) {
           req.user = {
             ...req.user,
             role:          'AGENT_VIEWER',
-            agent_uuid:    viewDecoded.viewAs,
+            agent_uuid:    targetAgent,
             agent_label:   viewDecoded.viewAsLabel,
             site_uuid:     targetSiteUuid,
             _viewAsMode:   true,
@@ -50,12 +78,25 @@ async function requireAuth(req, res, next) {
   }
 }
 
-function requireAdmin(req, res, next) {
+async function requireAdmin(req, res, next) {
   const token = req.cookies?.token;
   if (!token) return res.status(401).json({ error: 'Not authenticated' });
   try {
     const decoded = jwt.verify(token, JWT_SECRET);
-    if (decoded.role !== 'SUPER_ADMIN' && decoded.role !== 'TENANT_ADMIN' && decoded.role !== 'SOC_ANALYST') {
+
+    // Verify session status in MongoDB
+    if (decoded.session_id) {
+      const activeSession = await Session.findById(decoded.session_id);
+      if (!activeSession) {
+        res.clearCookie('token');
+        return res.status(401).json({ error: 'Sesi login telah dinonaktifkan atau kedaluwarsa.' });
+      }
+      activeSession.last_active = new Date();
+      await activeSession.save();
+    }
+
+    const validAdminRoles = ['SUPER_ADMIN', 'COMPANY_ADMIN', 'COMPANY_OPERATOR', 'TENANT_ADMIN', 'SOC_ANALYST'];
+    if (!validAdminRoles.includes(decoded.role)) {
       return res.status(403).json({ error: 'Forbidden' });
     }
     req.adminUser = decoded;

backend/models/Schemas.js

@@ -1,184 +1,187 @@
-// backend/models/Schemas.js
+// backend/models/Schemas.js
-// ─────────────────────────────────────────────────────────────────────────────
+// ─────────────────────────────────────────────────────────────────────────────
-// MongoDB Schemas untuk BackOne Backend (READ-ONLY)
+// MongoDB Schemas untuk BackOne Backend (READ-ONLY)
-//
+//
-// PENTING: Schema ini harus sinkron dengan proxy/models/Schemas.js
+// PENTING: Schema ini harus sinkron dengan proxy/models/Schemas.js
-// Proxy yang MENULIS data, backend yang MEMBACA data.
+// Proxy yang MENULIS data, backend yang MEMBACA data.
-//
+//
-// Setiap dokumen di-tag dengan:
+// Setiap dokumen di-tag dengan:
-//   agent_uuid  → identifikasi Network Agent spesifik (isolasi per tenant)
+//   agent_uuid  → identifikasi Network Agent spesifik (isolasi per tenant)
-//   site_uuid   → identifikasi site DPI (BackOne)
+//   site_uuid   → identifikasi site DPI (BackOne)
-//   timestamp   → waktu data dikumpulkan
+//   timestamp   → waktu data dikumpulkan
-// ─────────────────────────────────────────────────────────────────────────────
+// ─────────────────────────────────────────────────────────────────────────────
-
+
-const mongoose = require('mongoose');
+const mongoose = require('mongoose');
-
+
-const baseOptions = {
+const baseOptions = {
-  timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
+  timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
-};
+};
-
+
-// ─── Bandwidth Summary (per agent, per collection cycle) ───────────────────────
+// ─── Bandwidth Summary (per agent, per collection cycle) ───────────────────────
-const SummarySchema = new mongoose.Schema({
+const SummarySchema = new mongoose.Schema({
-  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
+  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid:     { type: String, index: true },   // null = global/all agents
+  agent_uuid:     { type: String, index: true },   // null = global/all agents
-  site_uuid:      { type: String, index: true },
+  site_uuid:      { type: String, index: true },
-  bandwidth_down: Number,
+  bandwidth_down: Number,
-  bandwidth_up:   Number,
+  bandwidth_up:   Number,
-  active_flows:   Number,
+  active_flows:   Number,
-  download_speed: Number,
+  download_speed: Number,
-  upload_speed:   Number,
+  upload_speed:   Number,
-  total_devices:  Number,
+  total_devices:  Number,
-  total_threats:  Number,
+  total_threats:  Number,
-  packet_drops:   Number,
+  packet_drops:   Number,
-  peak_flow_rate: Number,
+  peak_flow_rate: Number,
-  cpu_usage:      Number,
+  cpu_usage:      Number,
-  memory_usage:   Number,
+  memory_usage:   Number,
-  queue_depth:    Number,
+  queue_depth:    Number,
-}, baseOptions);
+}, baseOptions);
-
+
-// ─── Top Applications (per agent) ─────────────────────────────────────────────
+// ─── Top Applications (per agent) ─────────────────────────────────────────────
-const AppStatSchema = new mongoose.Schema({
+const AppStatSchema = new mongoose.Schema({
-  timestamp:  { type: Date,   required: true, index: true, expires: '7d' },
+  timestamp:  { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid: { type: String, index: true },
+  agent_uuid: { type: String, index: true },
-  site_uuid:  { type: String, index: true },
+  site_uuid:  { type: String, index: true },
-  app_label:  { type: String, required: true },
+  app_label:  { type: String, required: true },
-  download:   Number,
+  download:   Number,
-  upload:     Number,
+  upload:     Number,
-  flows:      Number,
+  flows:      Number,
-}, baseOptions);
+}, baseOptions);
-
+
-// ─── Protocol Statistics (per agent) ──────────────────────────────────────────
+// ─── Protocol Statistics (per agent) ──────────────────────────────────────────
-const ProtocolStatSchema = new mongoose.Schema({
+const ProtocolStatSchema = new mongoose.Schema({
-  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
+  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid:     { type: String, index: true },
+  agent_uuid:     { type: String, index: true },
-  site_uuid:      { type: String, index: true },
+  site_uuid:      { type: String, index: true },
-  protocol_label: { type: String, required: true },
+  protocol_label: { type: String, required: true },
-  download:       Number,
+  download:       Number,
-  upload:         Number,
+  upload:         Number,
-  flows:          Number,
+  flows:          Number,
-}, baseOptions);
+}, baseOptions);
-
+
-// ─── Discovered Devices (per agent, includes IP + MAC + device info) ───────────
+// ─── Discovered Devices (per agent, includes IP + MAC + device info) ───────────
-const DeviceStatSchema = new mongoose.Schema({
+const DeviceStatSchema = new mongoose.Schema({
-  timestamp:    { type: Date,   required: true, index: true, expires: '7d' },
+  timestamp:    { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid:   { type: String, index: true },
+  agent_uuid:   { type: String, index: true },
-  site_uuid:    { type: String, index: true },
+  site_uuid:    { type: String, index: true },
-  ip_address:   { type: String, required: true, index: true },
+  ip_address:   { type: String, required: true, index: true },
-  mac_address:  { type: String, index: true },
+  mac_address:  { type: String, index: true },
-  device_label: String,
+  device_label: String,
-  device_type:  String,
+  device_type:  String,
-  os_label:     String,
+  os_label:     String,
-  manufacturer: String,
+  manufacturer: String,
-  download:     Number,
+  download:     Number,
-  upload:       Number,
+  upload:       Number,
-  flows:        Number,
+  flows:        Number,
-  last_seen:    String,
+  last_seen:    String,
-}, baseOptions);
+}, baseOptions);
-
+
-// ─── Network Flows (per agent) ─────────────────────────────────────────────────
+// ─── Network Flows (per agent) ─────────────────────────────────────────────────
-const FlowSchema = new mongoose.Schema({
+const FlowSchema = new mongoose.Schema({
-  timestamp:  { type: Date,   required: true, index: true, expires: '7d' },
+  timestamp:  { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid: { type: String, index: true },
+  agent_uuid: { type: String, index: true },
-  site_uuid:  { type: String, index: true },
+  site_uuid:  { type: String, index: true },
-  flow_id:    String,
+  flow_id:    String,
-  src_ip:     { type: String, index: true },
+  src_ip:     { type: String, index: true },
-  src_mac:    String,
+  src_mac:    { type: String, index: true },
-  dst_ip:     { type: String, index: true },
+  dst_ip:     { type: String, index: true },
-  dst_port:   Number,
+  dst_port:   Number,
-  protocol:   String,
+  protocol:   String,
-  app_label:  String,
+  app_label:  String,
-  domain:     { type: String, index: true },
+  domain:     { type: String, index: true },
-  download:   Number,
+  download:   Number,
-  upload:     Number,
+  upload:     Number,
-  first_seen: String,
+  first_seen: String,
-  last_seen:  String,
+  last_seen:  String,
-}, baseOptions);
+}, baseOptions);
-
+
-// ─── Cyber Threats (per agent) ─────────────────────────────────────────────────
+// ─── Cyber Threats (per agent) ─────────────────────────────────────────────────
-const ThreatSchema = new mongoose.Schema({
+const ThreatSchema = new mongoose.Schema({
-  timestamp:   { type: Date,   required: true, index: true, expires: '7d' },
+  timestamp:   { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid:  { type: String, index: true },
+  agent_uuid:  { type: String, index: true },
-  site_uuid:   { type: String, index: true },
+  site_uuid:   { type: String, index: true },
-  threat_type: String,
+  threat_type: String,
-  severity:    String,
+  severity:    String,
-  src_ip:      String,
+  src_ip:      String,
-  dst_ip:      String,
+  dst_ip:      String,
-  dst_port:    Number,
+  dst_port:    Number,
-  protocol:    String,
+  protocol:    String,
-  description: String,
+  description: String,
-  event_at:    String,
+  event_at:    String,
-}, baseOptions);
+  flow_id:     { type: String, index: true },
-
+}, baseOptions);
-// ─── App Categories (per agent) ───────────────────────────────────────────────
+
-const AppCategoryStatSchema = new mongoose.Schema({
+// ─── App Categories (per agent) ───────────────────────────────────────────────
-  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
+const AppCategoryStatSchema = new mongoose.Schema({
-  agent_uuid:     { type: String, index: true },
+  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
-  site_uuid:      { type: String, index: true },
+  agent_uuid:     { type: String, index: true },
-  category_label: { type: String, required: true },
+  site_uuid:      { type: String, index: true },
-  download:       Number,
+  category_label: { type: String, required: true },
-  upload:         Number,
+  download:       Number,
-  flows:          Number,
+  upload:         Number,
-}, baseOptions);
+  flows:          Number,
-
+}, baseOptions);
-// ─── System Events (per agent) ─────────────────────────────────────────────────
+
-const EventSchema = new mongoose.Schema({
+// ─── System Events (per agent) ─────────────────────────────────────────────────
-  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
+const EventSchema = new mongoose.Schema({
-  agent_uuid:     { type: String, index: true },
+  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
-  site_uuid:      { type: String, index: true },
+  agent_uuid:     { type: String, index: true },
-  event_id:       Number,
+  site_uuid:      { type: String, index: true },
-  event_type:     String,
+  event_id:       Number,
-  severity:       String,
+  event_type:     String,
-  description:    String,
+  severity:       String,
-  category_label: String,
+  description:    String,
-  ip_address:     String,
+  category_label: String,
-  mac_address:    String,
+  ip_address:     String,
-  event_at:       Date,
+  mac_address:    String,
-}, baseOptions);
+  event_at:       Date,
-
+  flow_id:        { type: String, index: true },
-// ─── Compound Indexes for common dashboard queries ─────────────────────────────
+}, baseOptions);
-SummarySchema.index({ agent_uuid: 1, timestamp: -1 });
+
-AppStatSchema.index({ agent_uuid: 1, timestamp: -1, download: -1 });
+// ─── Compound Indexes for common dashboard queries ─────────────────────────────
-DeviceStatSchema.index({ agent_uuid: 1, ip_address: 1 }, { unique: true });
+SummarySchema.index({ agent_uuid: 1, timestamp: -1 });
-FlowSchema.index({ agent_uuid: 1, timestamp: -1 });
+AppStatSchema.index({ agent_uuid: 1, timestamp: -1, download: -1 });
-FlowSchema.index({ agent_uuid: 1, flow_id: 1 });
+DeviceStatSchema.index({ agent_uuid: 1, ip_address: 1 }, { unique: true });
-FlowSchema.index({ agent_uuid: 1, protocol: 1, timestamp: -1 });
+FlowSchema.index({ agent_uuid: 1, timestamp: -1 });
-FlowSchema.index({ agent_uuid: 1, domain: 1, timestamp: -1 });
+FlowSchema.index({ agent_uuid: 1, flow_id: 1 });
-FlowSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
+FlowSchema.index({ agent_uuid: 1, protocol: 1, timestamp: -1 });
-ThreatSchema.index({ agent_uuid: 1, timestamp: -1 });
+FlowSchema.index({ agent_uuid: 1, domain: 1, timestamp: -1 });
-AppCategoryStatSchema.index({ agent_uuid: 1, timestamp: -1 });
+FlowSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
-EventSchema.index({ agent_uuid: 1, timestamp: -1 });
+FlowSchema.index({ site_uuid: 1, src_mac: 1, timestamp: -1 });
-
+ThreatSchema.index({ agent_uuid: 1, timestamp: -1 });
-// ── Per-Device Per-Application Stats (synced from proxy) ─────────────────
+AppCategoryStatSchema.index({ agent_uuid: 1, timestamp: -1 });
-const DeviceAppStatSchema = new mongoose.Schema({
+EventSchema.index({ agent_uuid: 1, timestamp: -1 });
-  timestamp:  { type: Date,   required: true, index: true, expires: '7d' },
+
-  agent_uuid: { type: String, index: true },
+// ── Per-Device Per-Application Stats (synced from proxy) ─────────────────
-  site_uuid:  { type: String, index: true },
+const DeviceAppStatSchema = new mongoose.Schema({
-  ip_address: { type: String, required: true, index: true },
+  timestamp:  { type: Date,   required: true, index: true, expires: '7d' },
-  app_label:  { type: String, required: true },
+  agent_uuid: { type: String, index: true },
-  app_id:     Number,
+  site_uuid:  { type: String, index: true },
-  download:   { type: Number, default: 0 },
+  ip_address: { type: String, required: true, index: true },
-  upload:     { type: Number, default: 0 },
+  app_label:  { type: String, required: true },
-  flows:      { type: Number, default: 0 },
+  app_id:     Number,
-  last_seen:  String,
+  download:   { type: Number, default: 0 },
-}, baseOptions);
+  upload:     { type: Number, default: 0 },
-DeviceAppStatSchema.index({ agent_uuid: 1, ip_address: 1, timestamp: -1 });
+  flows:      { type: Number, default: 0 },
-DeviceAppStatSchema.index({ ip_address: 1, app_label: 1, timestamp: -1 });
+  last_seen:  String,
-DeviceAppStatSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
+}, baseOptions);
-
+DeviceAppStatSchema.index({ agent_uuid: 1, ip_address: 1, timestamp: -1 });
-const telemetrySchemas = require('./SchemasTelemetry');
+DeviceAppStatSchema.index({ ip_address: 1, app_label: 1, timestamp: -1 });
-const auxSchemas = require('./SchemasAux');
+DeviceAppStatSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
-
+
-module.exports = {
+const telemetrySchemas = require('./SchemasTelemetry');
-  Summary:       mongoose.model('Summary',       SummarySchema),
+const auxSchemas = require('./SchemasAux');
-  AppStat:       mongoose.model('AppStat',       AppStatSchema),
+
-  ProtocolStat:  mongoose.model('ProtocolStat',  ProtocolStatSchema),
+module.exports = {
-  DeviceStat:    mongoose.model('DeviceStat',    DeviceStatSchema),
+  Summary:       mongoose.model('Summary',       SummarySchema),
-  DeviceAppStat: mongoose.model('DeviceAppStat', DeviceAppStatSchema),
+  AppStat:       mongoose.model('AppStat',       AppStatSchema),
-  Flow:          mongoose.model('Flow',          FlowSchema),
+  ProtocolStat:  mongoose.model('ProtocolStat',  ProtocolStatSchema),
-  Threat:        mongoose.model('Threat',        ThreatSchema),
+  DeviceStat:    mongoose.model('DeviceStat',    DeviceStatSchema),
-  AppCategoryStat: mongoose.model('AppCategoryStat', AppCategoryStatSchema),
+  DeviceAppStat: mongoose.model('DeviceAppStat', DeviceAppStatSchema),
-  Event:         mongoose.model('Event',         EventSchema),
+  Flow:          mongoose.model('Flow',          FlowSchema),
-  ...auxSchemas,
+  Threat:        mongoose.model('Threat',        ThreatSchema),
-  ...telemetrySchemas
+  AppCategoryStat: mongoose.model('AppCategoryStat', AppCategoryStatSchema),
-};
+  Event:         mongoose.model('Event',         EventSchema),
-
+  ...auxSchemas,
+  ...telemetrySchemas
+};
+

backend/models/Session.js

@@ -0,0 +1,22 @@
+// backend/models/Session.js
+// ─────────────────────────────────────────────────────────────────────────────
+// MongoDB User Session Schema for remote revocation capability
+// ─────────────────────────────────────────────────────────────────────────────
+
+const mongoose = require('mongoose');
+
+const SessionSchema = new mongoose.Schema({
+  user_id: { type: mongoose.Schema.Types.ObjectId, ref: 'User', required: true, index: true },
+  ip_address: { type: String, default: 'Unknown' },
+  user_agent: { type: String, default: 'Unknown' },
+  session_token: { type: String, required: true, unique: true }, // JWT JTI or unique token hash
+  last_active: { type: Date, default: Date.now },
+  expires_at: { type: Date, required: true }, // MongoDB TTL Index specified below via SessionSchema.index
+}, {
+  timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
+});
+
+// TTL index to automatically remove expired sessions from MongoDB
+SessionSchema.index({ expires_at: 1 }, { expireAfterSeconds: 0 });
+
+module.exports = mongoose.model('Session', SessionSchema);

backend/models/User.js

@@ -14,11 +14,15 @@ const UserSchema = new mongoose.Schema({
   password_hash:   { type: String, required: true },
   account_name:    { type: String, default: null },
   profile_picture: { type: String, default: null },
-  role:            { type: String, enum: ['SUPER_ADMIN', 'TENANT_ADMIN', 'SOC_ANALYST', 'ENGINEER', 'AGENT_VIEWER'], default: 'AGENT_VIEWER' },
+  role:            { type: String, enum: ['SUPER_ADMIN', 'EXECUTIVE', 'TENANT_ADMIN', 'SOC_ANALYST', 'ENGINEER', 'AGENT_VIEWER', 'COMPANY_ADMIN', 'COMPANY_OPERATOR', 'COMPANY_VIEWER'], default: 'AGENT_VIEWER' },
   site_uuid:       { type: String, default: null, index: true },
   agent_uuid:      { type: String, default: null },
+  company_name:    { type: String, default: null, index: true },
+  agent_uuids:     { type: [String], default: [] },
   created_by:      { type: String, default: null, index: true },
   is_active:       { type: Boolean, default: true },
+  login_attempts:  { type: Number, default: 0 },
+  lockout_until:   { type: Date, default: null },
 }, {
   timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
 });

backend/scheduler.js

@@ -0,0 +1,312 @@
+// backend/scheduler.js
+const cron   = require('node-cron');
+const netify = require('./netify');
+const db     = require('./database');
+const SITE_UUID = process.env.NETIFY_SITE_UUID || 'dummy_site_uuid';
+
+let isRunning = false;
+
+async function runPoll() {
+  if (isRunning) {
+    console.log('[Scheduler] Poll sedang berjalan, skip.');
+    return;
+  }
+  isRunning = true;
+  const fetchedAt = new Date().toISOString();
+  console.log(`[Scheduler] Mulai polling... (${fetchedAt})`);
+
+  try {
+    // 0. Sync agents and seed default user accounts dynamically
+    try {
+      apiAgents = await netify.fetchAgents();
+      if (apiAgents && apiAgents.length > 0) {
+        db.syncAgentUsers(apiAgents);
+        console.log(`[Scheduler] OK Sync Agents  : ${apiAgents.length} agen terdeteksi`);
+      }
+    } catch (err) {
+      console.error('[Scheduler] Gagal sync agent users:', err.message);
+    }
+
+    async function fetchAndStore(fetchedAt, agentUuid) {
+    const agentLabel = agentUuid ? agentUuid : 'Global';
+    console.log(`[Scheduler] Fetching data for ${agentLabel}`);
+// 1. Top Aplikasi
+    const apps = await netify.fetchTopApps(1440, 20, agentUuid);
+    if (apps && Array.isArray(apps)) {
+      db.insertBandwidthApps(apps, fetchedAt, SITE_UUID, agentUuid);
+      console.log(`[Scheduler] OK Apps       : ${apps.length} baris`);
+    } else {
+      console.log(`[Scheduler] -- Apps       : tidak ada data`);
+    }
+
+    // 2. Top Devices — pakai fetchDiscoveredDevices yg sudah dinormalisasi
+    const devices = await netify.fetchDiscoveredDevices(1440, 200, agentUuid);
+    if (devices && Array.isArray(devices)) {
+      db.insertDevices(devices, fetchedAt, SITE_UUID, agentUuid);
+      console.log(`[Scheduler] OK Devices    : ${devices.length} baris`);
+    } else {
+      console.log(`[Scheduler] -- Devices    : tidak ada data`);
+    }
+
+    // 3. Top Protokol
+    const protocols = await netify.fetchTopProtocols(1440, 20, agentUuid);
+    if (protocols && Array.isArray(protocols)) {
+      db.insertProtocols(protocols, fetchedAt, SITE_UUID, agentUuid);
+      console.log(`[Scheduler] OK Protocols  : ${protocols.length} baris`);
+    } else {
+      console.log(`[Scheduler] -- Protocols  : tidak ada data`);
+    }
+
+    // 4. Top Negara
+    const countries = await netify.fetchTopCountries(1440, 15, agentUuid);
+    if (countries && Array.isArray(countries)) {
+      db.insertCountries(countries, fetchedAt, SITE_UUID, agentUuid);
+      console.log(`[Scheduler] OK Countries  : ${countries.length} baris`);
+    } else {
+      console.log(`[Scheduler] -- Countries  : tidak ada data`);
+    }
+
+    // 5. Top Domain/DNS
+    const domains = await netify.fetchTopDomains(1440, 20, agentUuid);
+    if (domains && Array.isArray(domains)) {
+      db.insertDNS(domains, fetchedAt, SITE_UUID, agentUuid);
+      console.log(`[Scheduler] OK DNS        : ${domains.length} baris`);
+    } else {
+      console.log(`[Scheduler] -- DNS        : tidak ada data`);
+    }
+
+    // 6. Flows — pakai local_ip sebagai proxy
+    const flows = await netify.fetchFlows(200, agentUuid);
+    if (flows && Array.isArray(flows)) {
+      db.insertFlows(flows, fetchedAt, SITE_UUID, agentUuid);
+      console.log(`[Scheduler] OK Flows      : ${flows.length} baris`);
+    } else {
+      console.log(`[Scheduler] -- Flows      : tidak ada data`);
+    }
+
+    // 7. Threats — dari Events Status
+    const threats = await netify.fetchCyberThreats(1440, 50, agentUuid);
+    if (threats && Array.isArray(threats)) {
+      db.insertThreats(threats, fetchedAt, SITE_UUID, agentUuid);
+      console.log(`[Scheduler] OK Threats    : ${threats.length} baris`);
+    } else {
+      console.log(`[Scheduler] -- Threats    : tidak ada data`);
+    }
+
+    // 8. Events Log
+    const events = await netify.fetchEvents(50, agentUuid);
+    if (events && Array.isArray(events)) {
+      db.insertEvents(events, fetchedAt, SITE_UUID, agentUuid);
+      console.log(`[Scheduler] OK Events     : ${events.length} baris`);
+    } else {
+      console.log(`[Scheduler] -- Events     : tidak ada data`);
+    }
+
+    // 10. App Categories
+    const appCats = await netify.fetchTopAppCategories(1440, 15, agentUuid);
+    if (appCats?.length) { db.insertAppCategories(appCats, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK AppCats    : ${appCats.length} baris`); }
+    else console.log(`[Scheduler] -- AppCats    : tidak ada data`);
+
+    // 11. Continents
+    const continents = await netify.fetchTopContinents(1440, 10, agentUuid);
+    if (continents?.length) { db.insertContinents(continents, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK Continents : ${continents.length} baris`); }
+    else console.log(`[Scheduler] -- Continents : tidak ada data`);
+
+    // 12. Regions
+    const regions = await netify.fetchTopRegions(1440, 20, agentUuid);
+    if (regions?.length) { db.insertRegions(regions, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK Regions    : ${regions.length} baris`); }
+    else console.log(`[Scheduler] -- Regions    : tidak ada data`);
+
+    // 13. Cities
+    const cities = await netify.fetchTopCities(1440, 20, agentUuid);
+    if (cities?.length) { db.insertCities(cities, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK Cities     : ${cities.length} baris`); }
+    else console.log(`[Scheduler] -- Cities     : tidak ada data`);
+
+    // 14. VLANs
+    const vlans = await netify.fetchTopVLANs(1440, 20, agentUuid);
+    if (vlans?.length) { db.insertVLANs(vlans, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK VLANs      : ${vlans.length} baris`); }
+    else console.log(`[Scheduler] -- VLANs      : tidak ada data`);
+
+    // 15. Interfaces
+    const ifaces = await netify.fetchTopInterfaces(1440, 20, agentUuid);
+    if (ifaces?.length) { db.insertInterfaces(ifaces, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK Interfaces : ${ifaces.length} baris`); }
+    else console.log(`[Scheduler] -- Interfaces : tidak ada data`);
+
+    // 16. Flow Types
+    const flowTypes = await netify.fetchTopFlowTypes(1440, 10, agentUuid);
+    if (flowTypes?.length) { db.insertFlowTypes(flowTypes, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK FlowTypes  : ${flowTypes.length} baris`); }
+    else console.log(`[Scheduler] -- FlowTypes  : tidak ada data`);
+
+    // 17. Flow Origins
+    const flowOrigins = await netify.fetchTopFlowOrigins(1440, 10, agentUuid);
+    if (flowOrigins?.length) { db.insertFlowOrigins(flowOrigins, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK FlowOrigin : ${flowOrigins.length} baris`); }
+    else console.log(`[Scheduler] -- FlowOrigin : tidak ada data`);
+
+    // 18. IP Versions
+    const ipVersions = await netify.fetchTopIPVersions(1440, 5, agentUuid);
+    if (ipVersions?.length) { db.insertIPVersions(ipVersions, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK IPVersions : ${ipVersions.length} baris`); }
+    else console.log(`[Scheduler] -- IPVersions : tidak ada data`);
+
+    // 19. Remote IPs
+    const remoteIPs = await netify.fetchTopRemoteIPs(1440, 20, agentUuid);
+    if (remoteIPs?.length) { db.insertRemoteIPs(remoteIPs, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK RemoteIPs  : ${remoteIPs.length} baris`); }
+    else console.log(`[Scheduler] -- RemoteIPs  : tidak ada data`);
+
+    // 20. MAC Bandwidth
+    const macBW = await netify.fetchTopLocalMACs(1440, 50, agentUuid);
+    if (macBW?.length) { db.insertMACBandwidth(macBW, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK MACBandwdh : ${macBW.length} baris`); }
+    else console.log(`[Scheduler] -- MACBandwdh : tidak ada data`);
+
+    // 9. Bandwidth Timeline
+    const summary  = await netify.fetchBandwidthSummary(1440, agentUuid);
+    const devCount = devices?.length ?? 0;
+    if (summary) {
+      db.insertBandwidthTimeline({ ...summary, devices: devCount }, fetchedAt, SITE_UUID, agentUuid);
+      console.log(`[Scheduler] OK Timeline   : saved`);
+    } else {
+      console.log(`[Scheduler] -- Timeline   : gagal ambil data`);
+    }
+
+    // 21. TLS Versions
+    const tlsVer = await netify.fetchTLSVersions(1440, 10, agentUuid);
+    if (tlsVer?.length) { db.insertTLSVersions(tlsVer, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK TLS Ver    : ${tlsVer.length} baris`); }
+    else console.log(`[Scheduler] -- TLS Ver    : tidak ada data`);
+
+    // 22. TLS Ciphers
+    const tlsCipher = await netify.fetchTLSCiphers(1440, 15, agentUuid);
+    if (tlsCipher?.length) { db.insertTLSCiphers(tlsCipher, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK TLS Cipher : ${tlsCipher.length} baris`); }
+    else console.log(`[Scheduler] -- TLS Cipher : tidak ada data`);
+
+    // 23. TLS Security
+    const tlsSec = await netify.fetchTLSSecurity(1440, 10, agentUuid);
+    if (tlsSec?.length) { db.insertTLSSecurity(tlsSec, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK TLS Sec    : ${tlsSec.length} baris`); }
+    else console.log(`[Scheduler] -- TLS Sec    : tidak ada data`);
+
+    // 24. NetBIOS Hostnames
+    const netbios = await netify.fetchNetBIOSHostnames(1440, 30, agentUuid);
+    if (netbios?.length) { db.insertNetBIOSHostnames(netbios, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK NetBIOS    : ${netbios.length} baris`); }
+    else console.log(`[Scheduler] -- NetBIOS    : tidak ada data`);
+
+    // 25. Discovery OS (standalone — OS yang terdeteksi di jaringan)
+    const discOs = await netify.fetchTopDiscoveryOS(1440, 20, agentUuid);
+    if (discOs?.length) { db.insertDiscoveryOS(discOs, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK DiscOS     : ${discOs.length} baris`); }
+    else console.log(`[Scheduler] -- DiscOS     : tidak ada data`);
+
+    // 26. DHCP Class Fingerprint
+    const dhcpFp = await netify.fetchDHCPClassFingerprints(1440, 30, agentUuid);
+    if (dhcpFp?.length) { db.insertDHCPFingerprints(dhcpFp, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK DHCP FP    : ${dhcpFp.length} baris`); }
+    else console.log(`[Scheduler] -- DHCP FP    : tidak ada data`);
+
+    // 27. HTTP User-Agent
+    const userAgents = await netify.fetchHTTPUserAgents(1440, 30, agentUuid);
+    if (userAgents?.length) { db.insertHTTPUserAgents(userAgents, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK UserAgent  : ${userAgents.length} baris`); }
+    else console.log(`[Scheduler] -- UserAgent  : tidak ada data`);
+
+    // 28. HTTPS SNI Hostname
+    const sniHosts = await netify.fetchSNIHostnames(1440, 30, agentUuid);
+    if (sniHosts?.length) { db.insertSNIHostnames(sniHosts, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK SNI Host   : ${sniHosts.length} baris`); }
+    else console.log(`[Scheduler] -- SNI Host   : tidak ada data`);
+
+    // 29. SSL Server Common Name
+    const sslCN = await netify.fetchSSLServerCN(1440, 30, agentUuid);
+    if (sslCN?.length) { db.insertSSLServerCN(sslCN, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK SSL CN     : ${sslCN.length} baris`); }
+    else console.log(`[Scheduler] -- SSL CN     : tidak ada data`);
+
+    // 30. QUIC Hostname
+    const quicHosts = await netify.fetchQUICHostnames(1440, 30, agentUuid);
+    if (quicHosts?.length) { db.insertQUICHostnames(quicHosts, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK QUIC Host  : ${quicHosts.length} baris`); }
+    else console.log(`[Scheduler] -- QUIC Host  : tidak ada data`);
+
+    // 31. BitTorrent Info Hash
+    const btHashes = await netify.fetchBitTorrentInfoHashes(1440, 30, agentUuid);
+    if (btHashes?.length) { db.insertBitTorrentHashes(btHashes, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK BT Hash    : ${btHashes.length} baris`); }
+    else console.log(`[Scheduler] -- BT Hash    : tidak ada data`);
+
+    // 32. SSH Client (field: ssh_client)
+    const sshClient = await netify.fetchSSHClients(1440, 20, agentUuid);
+    if (sshClient?.length) { db.insertSSHVersions(sshClient, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK SSH Client : ${sshClient.length} baris`); }
+    else console.log(`[Scheduler] -- SSH Client : tidak ada data`);
+
+    // 32b. SSH Server (field: ssh_server)
+    const sshServer = await netify.fetchSSHServers(1440, 20, agentUuid);
+    if (sshServer?.length) { db.insertSSHVersions(sshServer, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK SSH Server : ${sshServer.length} baris`); }
+    else console.log(`[Scheduler] -- SSH Server : tidak ada data`);
+
+    // 33. mDNS Hostname (Chromecast, Apple TV, etc.)
+    const mdnsHosts = await netify.fetchMDNSHostnames(1440, 30, agentUuid);
+    if (mdnsHosts?.length) { db.insertMDNSHostnames(mdnsHosts, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK mDNS Host  : ${mdnsHosts.length} baris`); }
+    else console.log(`[Scheduler] -- mDNS Host  : tidak ada data`);
+
+    // ─── INTELLIGENCE 22-30 (derive dari data yang tersedia) ─────────────────
+
+    // 34. Cryptocurrency Mining (derive dari apps + flows ke port mining)
+    const cryptoMining = await netify.fetchCryptoMining(50, agentUuid);
+    if (cryptoMining?.length) { db.insertCryptoMining(cryptoMining, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK CryptoMine : ${cryptoMining.length} baris`); }
+    else console.log(`[Scheduler] -- CryptoMine : tidak ada data`);
+
+    // 35. Device Discovery (derive dari flows + bandwidth per-IP)
+    const devDisc = await netify.fetchDeviceDiscovery(100, agentUuid);
+    if (devDisc?.length) { db.insertDeviceDiscovery(devDisc, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK DevDisc    : ${devDisc.length} baris`); }
+    else console.log(`[Scheduler] -- DevDisc    : tidak ada data`);
+
+    // 36. Encryption Audit (derive dari flows per-IP: port encrypted vs plain)
+    const encAudit = await netify.fetchEncryptionAudit(50, agentUuid);
+    if (encAudit?.length) { db.insertEncryptionAudit(encAudit, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK EncAudit   : ${encAudit.length} baris`); }
+    else console.log(`[Scheduler] -- EncAudit   : tidak ada data`);
+
+    // 37. Insecure Protocols (derive dari top protocols)
+    const insecProto = await netify.fetchInsecureProtocols(1440, 50, agentUuid);
+    if (insecProto?.length) { db.insertInsecureProtocols(insecProto, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK InsecProto : ${insecProto.length} baris`); }
+    else console.log(`[Scheduler] -- InsecProto : tidak ada data`);
+
+    // 38. IP Reputation (derive dari top remote_ip + high-risk countries)
+    const ipRep = await netify.fetchIPReputation(50, agentUuid);
+    if (ipRep?.length) { db.insertIPReputation(ipRep, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK IPRepute   : ${ipRep.length} baris`); }
+    else console.log(`[Scheduler] -- IPRepute   : tidak ada data`);
+
+    // 39. Server Discovery (derive dari flows ke port server well-known)
+    const srvDisc = await netify.fetchServerDiscovery(100, agentUuid);
+    if (srvDisc?.length) { db.insertServerDiscovery(srvDisc, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK SrvDisc    : ${srvDisc.length} baris`); }
+    else console.log(`[Scheduler] -- SrvDisc    : tidak ada data`);
+
+    // 40. Tor Detection (derive dari apps/hostnames mengandung "tor")
+    const torDet = await netify.fetchTorDetection(50, agentUuid);
+    if (torDet?.length) { db.insertTorDetection(torDet, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK TorDet     : ${torDet.length} baris`); }
+    else console.log(`[Scheduler] -- TorDet     : tidak ada data`);
+
+    // 41. Unencrypted Password (derive dari flows ke port cleartext auth)
+    const unencPwd = await netify.fetchUnencryptedPasswords(50, agentUuid);
+    if (unencPwd?.length) { db.insertUnencryptedPasswords(unencPwd, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK UnencPwd   : ${unencPwd.length} baris`); }
+    else console.log(`[Scheduler] -- UnencPwd   : tidak ada data`);
+
+    // 42. VPN Detection (derive dari apps/protocols/ports VPN)
+    const vpnDet = await netify.fetchVPNDetection(50, agentUuid);
+    if (vpnDet?.length) { db.insertVPNDetection(vpnDet, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK VPNDet     : ${vpnDet.length} baris`); }
+    else console.log(`[Scheduler] -- VPNDet     : tidak ada data`);
+
+  }
+
+    // --- Main loop
+    await fetchAndStore(fetchedAt, null);
+    if (apiAgents && apiAgents.length > 0) {
+      for (const agent of apiAgents) {
+        if (agent && agent.uuid) {
+          await fetchAndStore(fetchedAt, agent.uuid);
+        }
+      }
+    }
+  } catch (err) {
+    console.error('[Scheduler] ERROR:', err);
+  } finally {
+    isRunning = false;
+    console.log(`[Scheduler] Poll selesai.\n`);
+  }
+}
+
+function startScheduler() {
+  runPoll();
+  cron.schedule('* * * * *', () => runPoll());
+  console.log('[Scheduler] Aktif. Polling setiap 1 menit.\n');
+}
+
+module.exports = { startScheduler, runPoll };

backend/server.js

@@ -1,123 +1,163 @@
-// backend/server.js
+// backend/server.js
-// ─────────────────────────────────────────────────────────────────────────────
+// ─────────────────────────────────────────────────────────────────────────────
-// BackOne Backend API Server
+// Polyfill global crypto for Node 18 compatibility (required by mongodb driver)
-//
+if (typeof globalThis.crypto === 'undefined') {
-// Tanggung jawab backend ini adalah READ-ONLY dari MongoDB.
+  globalThis.crypto = require('crypto');
-// Semua data collection (ingestion) dilakukan oleh Proxy Server (port 4000).
+}
-// Backend TIDAK memanggil DPI API secara langsung.
+
-//
+// BackOne Backend API Server
-// Environment Variables:
+//
-//   MONGODB_URI    - MongoDB connection string
+// Tanggung jawab backend ini adalah READ-ONLY dari MongoDB.
-//   BACKEND_PORT   - Port server ini (default: 3001)
+// Semua data collection (ingestion) dilakukan oleh Proxy Server (port 4000).
-//   JWT_SECRET     - Secret untuk JWT auth
+// Backend TIDAK memanggil DPI API secara langsung.
-//   ALLOWED_ORIGINS- Comma-separated allowed CORS origins
+//
-//   PROXY_URL      - URL proxy server (untuk trigger manual refresh)
+// Environment Variables:
-// ─────────────────────────────────────────────────────────────────────────────
+//   MONGODB_URI    - MongoDB connection string
-
+//   BACKEND_PORT   - Port server ini (default: 3001)
-const path = require('path');
+//   JWT_SECRET     - Secret untuk JWT auth
-require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
+//   ALLOWED_ORIGINS- Comma-separated allowed CORS origins
-
+//   PROXY_URL      - URL proxy server (untuk trigger manual refresh)
-const express     = require('express');
+// ─────────────────────────────────────────────────────────────────────────────
-const cors        = require('cors');
+
-const cookieParser = require('cookie-parser');
+const path = require('path');
-const jwt         = require('jsonwebtoken');
+const envFile = process.env.NODE_ENV === 'production' ? '.env.production' : '.env.local';
-const connectDB   = require('./db/mongoose');
+require('dotenv').config({ path: path.join(__dirname, '..', envFile) });
-
+
-// ─── Connect to MongoDB (read-only mode) ──────────────────────────────────────
+const express     = require('express');
-connectDB();
+const cors        = require('cors');
-
+const cookieParser = require('cookie-parser');
-const app  = express();
+const jwt         = require('jsonwebtoken');
-const PORT = process.env.BACKEND_PORT || 3001;
+const connectDB   = require('./db/mongoose');
-
+
-// ─── Middleware ────────────────────────────────────────────────────────────────
+// ─── Connect to MongoDB (read-only mode) ──────────────────────────────────────
-const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS
+connectDB();
-  ? process.env.ALLOWED_ORIGINS.split(',')
+
-  : ['http://localhost:3000', 'http://127.0.0.1:3000'];
+const app  = express();
-
+const PORT = process.env.BACKEND_PORT || 3001;
-app.use(cors({
+
-  origin: (origin, callback) => {
+// ─── Middleware ────────────────────────────────────────────────────────────────
-    if (!origin) return callback(null, true);
+const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS
-    if (ALLOWED_ORIGINS.includes(origin)) {
+  ? process.env.ALLOWED_ORIGINS.split(',')
-      callback(null, true);
+  : ['http://localhost:3000', 'http://127.0.0.1:3000'];
-    } else {
+
-      callback(new Error('Blocked by CORS policy (Unauthorized Origin)'));
+app.use(cors({
-    }
+  origin: (origin, callback) => {
-  },
+    if (!origin) return callback(null, true);
-  credentials: true
+    if (ALLOWED_ORIGINS.includes(origin)) {
-}));
+      callback(null, true);
-app.use(express.json());
+    } else {
-app.use(cookieParser());
+      callback(new Error('Blocked by CORS policy (Unauthorized Origin)'));
-
+    }
-// ─── Public Routes ────────────────────────────────────────────────────────────
+  },
-const authRoutes = require('./routes/auth');
+  credentials: true
-const { getUploadsDir } = require('./routes/auth/helpers');
+}));
-app.use('/api/auth', authRoutes);
+app.use(express.json({ limit: '10mb' }));
-app.use('/api/uploads', express.static(getUploadsDir()));
+app.use(express.urlencoded({ extended: true, limit: '10mb' }));
-
+app.use(cookieParser());
-// ─── Auth Middleware ──────────────────────────────────────────────────────────
+
-const { requireAuth } = require('./middleware/auth');
+app.use((req, res, next) => {
-const { getTimeFilter, getBaseFilter } = require('./routes/dashboard/helpers');
+  if (req.originalUrl && req.originalUrl.includes('/api/dashboard')) {
-const {
+    try {
-  generateMacFromIp,
+      const fs = require('fs');
-  resolveVendorFromIp,
+      const path = require('path');
-  resolveDeviceTypeFromIp,
+      const logPath = path.join(__dirname, '../scratch/http_requests.log');
-  resolveOSFromIp,
+      const logLine = `[${new Date().toISOString()}] ${req.method} ${req.originalUrl} - Query: ${JSON.stringify(req.query)}\n`;
-  generateAutoLabel
+      fs.appendFileSync(logPath, logLine);
-} = require('./deviceResolver');
+    } catch (e) {
-
+      console.error('Logger error:', e.message);
-// ─── Protected Dashboard Routes ───────────────────────────────────────────────
+    }
-const dashboardRoutes = require('./routes/dashboard');
+  }
-
+  next();
-// Override /api/dashboard/app-details to show real-time device mapping per application
+});
-app.get('/api/dashboard/app-details', requireAuth, (req, res) => {
+
-  require('./routes/appDetailsHandler')(req, res, {
+
-    getTimeFilter,
+// ─── Public Routes ────────────────────────────────────────────────────────────
-    getBaseFilter
+const authRoutes = require('./routes/auth');
-  });
+const { getUploadsDir } = require('./routes/auth/helpers');
-});
+app.use('/api/auth', authRoutes);
-
+app.use('/api/uploads', express.static(getUploadsDir()));
-// Override /api/dashboard/device-details to map real-time classifications (Facebook, YouTube, etc.)
+
-app.get('/api/dashboard/device-details', requireAuth, (req, res) => {
+// ─── Auth Middleware ──────────────────────────────────────────────────────────
-  require('./routes/deviceDetailsHandler')(req, res, {
+const { requireAuth } = require('./middleware/auth');
-    getTimeFilter,
+const { getTimeFilter, getBaseFilter } = require('./routes/dashboard/helpers');
-    getBaseFilter,
+const {
-    generateMacFromIp,
+  generateMacFromIp,
-    resolveDeviceTypeFromIp,
+  resolveVendorFromIp,
-    resolveOSFromIp,
+  resolveDeviceTypeFromIp,
-    resolveVendorFromIp,
+  resolveOSFromIp,
-    generateAutoLabel
+  generateAutoLabel
-  });
+} = require('./deviceResolver');
-});
+
-
+// ─── Protected Dashboard Routes ───────────────────────────────────────────────
-app.get('/api/dashboard/remote-ip-details', requireAuth, async (req, res) => {
+const dashboardRoutes = require('./routes/dashboard');
-  require('./routes/remoteIpDetailsHandler')(req, res, {
+
-    getTimeFilter
+// Override /api/dashboard/app-details to show real-time device mapping per application
-  });
+app.get('/api/dashboard/app-details', requireAuth, (req, res) => {
-});
+  require('./routes/appDetailsHandler')(req, res, {
-
+    getTimeFilter,
-const metadataDetailRoutes = require('./routes/metadataDetail');
+    getBaseFilter
-app.use('/api/dashboard/metadata-detail', requireAuth, metadataDetailRoutes);
+  });
-
+});
-const categoryDetailRoutes = require('./routes/categoryDetail');
+
-app.use('/api/dashboard/category-detail', requireAuth, categoryDetailRoutes);
+// Override /api/dashboard/device-details to map real-time classifications (Facebook, YouTube, etc.)
-
+app.get('/api/dashboard/device-details', requireAuth, (req, res) => {
-app.use('/api/dashboard', requireAuth, dashboardRoutes);
+  require('./routes/deviceDetailsHandler')(req, res, {
-
+    getTimeFilter,
-
+    getBaseFilter,
-
+    generateMacFromIp,
-
+    resolveDeviceTypeFromIp,
-// ─── Health Check ─────────────────────────────────────────────────────────────
+    resolveOSFromIp,
-app.get('/api/health', (req, res) => {
+    resolveVendorFromIp,
-  res.json({
+    generateAutoLabel
-    ok: true,
+  });
-    message: 'BackOne Backend berjalan (MongoDB read-only mode)',
+});
-    time: new Date().toISOString()
+
-  });
+app.get('/api/dashboard/remote-ip-details', requireAuth, async (req, res) => {
-});
+  require('./routes/remoteIpDetailsHandler')(req, res, {
-
+    getTimeFilter
-// ─── Start Server ─────────────────────────────────────────────────────────────
+  });
-app.listen(PORT, () => {
+});
-  console.log(`\n🚀 BackOne API Server berjalan di http://localhost:${PORT}`);
+
-  console.log(`🔌 API Health : http://localhost:${PORT}/api/health`);
+const metadataDetailRoutes = require('./routes/metadataDetail');
-  console.log(`📡 Mode       : READ-ONLY dari MongoDB (data dikirim oleh Proxy Server)\n`);
+app.use('/api/dashboard/metadata-detail', requireAuth, metadataDetailRoutes);
-});
+
+const categoryDetailRoutes = require('./routes/categoryDetail');
+app.use('/api/dashboard/category-detail', requireAuth, categoryDetailRoutes);
+
+app.use('/api/dashboard', requireAuth, dashboardRoutes);
+
+
+
+
+// ─── Health Check ─────────────────────────────────────────────────────────────
+app.get('/api/health', (req, res) => {
+  res.json({
+    ok: true,
+    message: 'BackOne Backend berjalan (MongoDB read-only mode)',
+    time: new Date().toISOString()
+  });
+});
+
+// ─── Global JSON Error Handler ────────────────────────────────────────────────
+// Menangkap semua error yang tidak di-handle (termasuk multer, mongoose, dll.)
+// dan memastikan response selalu JSON, BUKAN HTML default Express.
+// eslint-disable-next-line no-unused-vars
+app.use((err, req, res, next) => {
+  console.error('[Global Error Handler]', err.message || err);
+  const status = err.status || err.statusCode || 500;
+  res.status(status).json({
+    error: err.message || 'Internal server error',
+    code: err.code || undefined,
+  });
+});
+
+// ─── Start Server ─────────────────────────────────────────────────────────────
+// Bind to 127.0.0.1 in production to prevent direct external access to port 3001.
+// All external traffic must go through the reverse proxy (Apache/Nginx) at port 80/443.
+const BIND_HOST = process.env.NODE_ENV === 'production' ? '127.0.0.1' : '0.0.0.0';
+app.listen(PORT, BIND_HOST, () => {
+  console.log(`\n🚀 BackOne API Server berjalan di http://${BIND_HOST}:${PORT}`);
+  console.log(`🔌 API Health : http://${BIND_HOST}:${PORT}/api/health`);
+  console.log(`📡 Mode       : READ-ONLY dari MongoDB (data dikirim oleh Proxy Server)`);
+  console.log(`🔒 Security   : Bound to ${BIND_HOST} (internal only in production)\n`);
+});

backend/services/ingestionService.js

@@ -0,0 +1,135 @@
+const cron = require('node-cron');
+const netify = require('../netify');
+const { Summary, AppStat, ProtocolStat, DeviceStat, Flow, Threat } = require('../models/Schemas');
+
+const SITE_UUID = process.env.NETIFY_SITE_UUID || process.env.BACKONE_SITE_UUID;
+let isRunning = false;
+
+async function runPoll() {
+  if (isRunning) return;
+  isRunning = true;
+  const timestamp = new Date();
+  console.log(`[Mongo-Ingestion] Started polling at ${timestamp.toISOString()}`);
+
+  try {
+    const agents = await netify.fetchAgents();
+    const agentList = agents && agents.length > 0 ? agents.map(a => a.uuid) : [null]; // null for global
+
+    for (const agentUuid of agentList) {
+      console.log(`[Mongo-Ingestion] Fetching data for Agent: ${agentUuid || 'Global'}`);
+
+      // 1. Summary
+      const summary = await netify.fetchBandwidthSummary(1440, agentUuid);
+      if (summary) {
+        await new Summary({
+          timestamp,
+          agent_uuid: agentUuid,
+          site_uuid: SITE_UUID,
+          ...summary
+        }).save();
+      }
+
+      // 2. Apps
+      const apps = await netify.fetchTopApps(1440, 200, agentUuid); // high limit for data lake
+      if (apps && apps.length > 0) {
+        const appDocs = apps.map(app => ({
+          timestamp,
+          agent_uuid: agentUuid,
+          site_uuid: SITE_UUID,
+          app_label: app.application?.label || 'Unknown',
+          download: app.download || 0,
+          upload: app.upload || 0,
+          flows: app.flows || 0
+        }));
+        await AppStat.insertMany(appDocs);
+      }
+
+      const devices = await netify.fetchDiscoveredDevices(1440, 500, agentUuid);
+      if (devices && devices.length > 0) {
+        const devDocs = devices.map(d => ({
+          timestamp,
+          agent_uuid: agentUuid,
+          site_uuid: SITE_UUID,
+          ip_address: d.ip_address,
+          mac_address: d.mac_address,
+          device_label: d.device_label,
+          device_type: d.device_type,
+          os_label: d.os_label,
+          manufacturer: d.manufacturer,
+          download: d.download || 0,
+          upload: d.upload || 0,
+          flows: d.flows || 0,
+          last_seen: d.last_seen
+        })).filter(d => d.ip_address); // Ensure ip_address exists to avoid validation error
+        if (devDocs.length > 0) {
+          await DeviceStat.insertMany(devDocs);
+        }
+      }
+      
+      // 4. Flows
+      const flows = await netify.fetchFlows(500, agentUuid);
+      if (flows && flows.length > 0) {
+        const flowDocs = flows.map(f => ({
+          timestamp,
+          agent_uuid: agentUuid,
+          site_uuid: SITE_UUID,
+          flow_id: f.flow_id,
+          src_ip: f.src_ip,
+          src_mac: f.src_mac,
+          dst_ip: f.dst_ip,
+          dst_port: f.dst_port,
+          protocol: f.protocol,
+          app_label: f.app_label,
+          domain: f.domain,
+          download: f.download || 0,
+          upload: f.upload || 0,
+          first_seen: f.first_seen,
+          last_seen: f.last_seen
+        })).filter(f => f.src_ip);
+        if (flowDocs.length > 0) {
+          await Flow.insertMany(flowDocs);
+        }
+      }
+
+      // 5. Threats
+      const threats = await netify.fetchCyberThreats(agentUuid);
+      if (threats && threats.length > 0) {
+        const threatDocs = threats.map(t => ({
+          timestamp,
+          agent_uuid: agentUuid,
+          site_uuid: SITE_UUID,
+          threat_type: t.threat_type || 'Unknown Threat',
+          severity: t.severity || 'Medium',
+          src_ip: t.src_ip,
+          dst_ip: t.dst_ip,
+          dst_port: t.dst_port,
+          protocol: t.protocol,
+          description: t.description,
+          event_at: t.event_at || new Date().toISOString()
+        }));
+        if (threatDocs.length > 0) {
+          await Threat.insertMany(threatDocs);
+        }
+      }
+    }
+  } catch (error) {
+    console.error('[Mongo-Ingestion] Error during polling:', error);
+  } finally {
+    isRunning = false;
+  }
+}
+
+function startScheduler() {
+  // Run every 5 minutes
+  cron.schedule('*/5 * * * *', () => {
+    runPoll();
+  });
+  console.log('[Mongo-Ingestion] Scheduler started (every 5 minutes)');
+  
+  // Initial run
+  runPoll();
+}
+
+module.exports = { startScheduler };
+
+

backend/test_api_tls.js

@@ -0,0 +1,14 @@
+const http = require('http');
+
+http.get('http://localhost:3001/api/dashboard/tls-versions', {
+  headers: {
+    'Cookie': 'token=test', // Just checking schema, if it requires auth we might need to mock or use the proxy
+  }
+}, (res) => {
+  let data = '';
+  res.on('data', chunk => data += chunk);
+  res.on('end', () => {
+    console.log("Response TLS Versions:");
+    console.log(data.slice(0, 500));
+  });
+});

check-mongo.js

@@ -0,0 +1,60 @@
+// check-mongo.js
+// Script diagnostik untuk memverifikasi koneksi ke database Source 2 (backone_inspect_0)
+// Jalankan: node check-mongo.js
+
+const path = require('path');
+require('dotenv').config({ path: path.join(__dirname, '.env.local') });
+
+const mongoose = require('mongoose');
+
+const MONGODB_URI = process.env.MONGODB_URI;
+
+if (!MONGODB_URI) {
+  console.error('[ERROR] MONGODB_URI tidak ditemukan di .env.local');
+  process.exit(1);
+}
+
+console.log('\n╔════════════════════════════════════════════════╗');
+console.log('║    Source 2 — MongoDB Connection Diagnostic   ║');
+console.log('╚════════════════════════════════════════════════╝\n');
+console.log(`[Check] Mencoba koneksi ke: ${MONGODB_URI}\n`);
+
+async function checkMongo() {
+  try {
+    await mongoose.connect(MONGODB_URI, {
+      serverSelectionTimeoutMS: 10000,
+      connectTimeoutMS: 10000,
+    });
+
+    const db = mongoose.connection.db;
+    const dbName = db.databaseName;
+
+    console.log(`[OK] Berhasil terhubung ke MongoDB!`);
+    console.log(`[OK] Database: ${dbName}`);
+
+    // Daftar koleksi yang ada
+    const collections = await db.listCollections().toArray();
+    if (collections.length === 0) {
+      console.log('[INFO] Database masih kosong — belum ada koleksi.');
+    } else {
+      console.log(`[INFO] Koleksi yang ada (${collections.length}):`);
+      for (const col of collections) {
+        const count = await db.collection(col.name).countDocuments();
+        console.log(`       - ${col.name}: ${count} dokumen`);
+      }
+    }
+
+    console.log('\n[RESULT] ✅ STEP 8 PASS — Koneksi ke database Source 2 berhasil.\n');
+    process.exit(0);
+  } catch (err) {
+    console.error(`[ERROR] Gagal terhubung ke MongoDB: ${err.message}`);
+    console.error('\nPossible causes:');
+    console.error('  1. Host "mongodb-netify" tidak bisa dijangkau (butuh VPN/SSH tunnel)');
+    console.error('  2. Kredensial backone_inspect:backone_inspect salah');
+    console.error('  3. MongoDB belum berjalan di server tujuan');
+    console.error('\n[RESULT] ❌ STEP 8 FAIL — Hubungi atasan untuk verifikasi koneksi.\n');
+    process.exit(1);
+  }
+}
+
+checkMongo();

deploy-server-setup.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+# =============================================================================
+# deploy-server-setup.sh
+# Script yang dijalankan di server setelah file di-upload
+# Path: /home/adminbackend/web/dev.demoplace.my.id/public_html/
+# =============================================================================
+
+set -e
+DEPLOY_DIR="/home/adminbackend/web/dev.demoplace.my.id/public_html"
+cd "$DEPLOY_DIR"
+
+echo "=== [1/6] Checking environment ==="
+node --version
+npm --version
+pm2 --version || npm install -g pm2
+
+echo ""
+echo "=== [2/6] Installing backend dependencies ==="
+cd "$DEPLOY_DIR/backend"
+npm install --omit=dev --legacy-peer-deps
+cd "$DEPLOY_DIR"
+
+echo ""
+echo "=== [3/6] Installing proxy dependencies ==="
+cd "$DEPLOY_DIR/proxy"
+npm install --omit=dev --legacy-peer-deps
+cd "$DEPLOY_DIR"
+
+echo ""
+echo "=== [4/6] Creating required directories ==="
+mkdir -p logs
+mkdir -p scratch
+
+echo ""
+echo "=== [5/6] Stopping old PM2 processes (if any) ==="
+pm2 delete source2-proxy   2>/dev/null || echo "source2-proxy: not running"
+pm2 delete source2-backend 2>/dev/null || echo "source2-backend: not running"
+pm2 delete source2-frontend 2>/dev/null || echo "source2-frontend: not running"
+
+echo ""
+echo "=== [6/6] Starting PM2 processes ==="
+pm2 start ecosystem.config.js --env production
+pm2 save
+pm2 list
+
+echo ""
+echo "=== DEPLOY COMPLETE ==="
+echo "Frontend  : http://127.0.0.1:3010"
+echo "Backend   : http://127.0.0.1:3011"
+echo "Proxy     : http://127.0.0.1:4010"
+echo ""
+echo "Check logs with: pm2 logs source2-backend --lines 30"

docker-compose.yml

@@ -53,12 +53,9 @@ services:
       - MONGODB_URI=mongodb://mongodb:27017/backone_dpi
       - PROXY_PORT=4000
       # Mode 1: kumpulkan SEMUA agent (default)
-      # Ubah ke PROXY_COLLECT_MODE=agent dan isi PROXY_AGENT_UUID untuk mode spesifik (1 agent)
+      # Ubah ke PROXY_COLLECT_MODE=agent dan isi PROXY_AGENT_UUID untuk mode spesifik
-      # Ubah ke PROXY_COLLECT_MODE=agents dan isi PROXY_AGENT_UUIDS untuk mode multi-agent
       - PROXY_COLLECT_MODE=${PROXY_COLLECT_MODE:-all}
       - PROXY_AGENT_UUID=${PROXY_AGENT_UUID:-}
-      - PROXY_AGENT_UUIDS=${PROXY_AGENT_UUIDS:-}
-      - PROXY_AGENT_DELAY_MS=${PROXY_AGENT_DELAY_MS:-5000}
       - PROXY_CRON_SCHEDULE=${PROXY_CRON_SCHEDULE:-*/5 * * * *}
     networks:
       - backone-infra

ecosystem.config.js

@@ -1,22 +1,64 @@
+// ecosystem.config.js — PM2 Configuration for Source 2 (dev.demoplace.my.id)
 module.exports = {
   apps: [
     {
-      name: "backone-proxy",
+      name: 'source2-proxy',
-      script: "./proxy/index.js",
+      script: './proxy/index.js',
-      env_file: ".env.production"
+      cwd: '/home/adminbackend/web/dev.demoplace.my.id/public_html',
+      instances: 1,
+      exec_mode: 'fork',
+      watch: false,
+      node_args: '--max-old-space-size=1024',
+      max_memory_restart: '1200M',
+      restart_delay: 5000,
+      max_restarts: 10,
+      env_file: '.env.production',
+      env: { NODE_ENV: 'production' },
+      error_file: './logs/proxy-error.log',
+      out_file: './logs/proxy-out.log',
+      log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
+      merge_logs: true,
     },
     {
-      name: "backone-backend",
+      name: 'source2-backend',
-      script: "./backend/server.js",
+      script: './backend/server.js',
-      env_file: ".env.production"
+      cwd: '/home/adminbackend/web/dev.demoplace.my.id/public_html',
+      instances: 1,
+      exec_mode: 'fork',
+      watch: false,
+      node_args: '--max-old-space-size=256',
+      max_memory_restart: '400M',
+      restart_delay: 3000,
+      max_restarts: 10,
+      env_file: '.env.production',
+      env: { NODE_ENV: 'production' },
+      error_file: './logs/backend-error.log',
+      out_file: './logs/backend-out.log',
+      log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
+      merge_logs: true,
     },
     {
-      name: "backone-frontend",
+      name: 'source2-frontend',
-      script: "server.js",
+      script: 'start-with-env.js',
-      env_file: ".env.production",
+      cwd: '/home/adminbackend/web/dev.demoplace.my.id/public_html',
+      instances: 1,
+      exec_mode: 'fork',
+      watch: false,
+      node_args: '--max-old-space-size=512',
+      max_memory_restart: '700M',
+      restart_delay: 3000,
+      max_restarts: 10,
+      env_file: '.env.production',
       env: {
-        PORT: 8009
+        NODE_ENV: 'production',
-      }
+        PORT: 3010,
-    }
+        HOSTNAME: '127.0.0.1',
-  ]
+        NEXT_TELEMETRY_DISABLED: '1',
+      },
+      error_file: './logs/frontend-error.log',
+      out_file: './logs/frontend-out.log',
+      log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
+      merge_logs: true,
+    },
+  ],
 };

eslint.config.mjs

@@ -1,21 +1,21 @@
-import { defineConfig, globalIgnores } from "eslint/config";
+import { defineConfig, globalIgnores } from "eslint/config";
-import nextVitals from "eslint-config-next/core-web-vitals";
+import nextVitals from "eslint-config-next/core-web-vitals";
-import nextTs from "eslint-config-next/typescript";
+import nextTs from "eslint-config-next/typescript";
-
+
-const eslintConfig = defineConfig([
+const eslintConfig = defineConfig([
-  ...nextVitals,
+  ...nextVitals,
-  ...nextTs,
+  ...nextTs,
-  // Override default ignores of eslint-config-next.
+  // Override default ignores of eslint-config-next.
-  globalIgnores([
+  globalIgnores([
-    // Default ignores of eslint-config-next:
+    // Default ignores of eslint-config-next:
-    ".next/**",
+    ".next/**",
-    "out/**",
+    "out/**",
-    "build/**",
+    "build/**",
-    "next-env.d.ts",
+    "next-env.d.ts",
-    "backend/**",
+    "backend/**",
-    "proxy/**",
+    "proxy/**",
-    "test/**",
+    "test/**",
-  ]),
+  ]),
-]);
+]);
-
+
-export default eslintConfig;
+export default eslintConfig;

git-push-iso.js

@@ -0,0 +1,259 @@
+/**
+ * git-push-iso.js
+ * Push ke Gitea & GitHub menggunakan isomorphic-git (pure JS, tanpa system git)
+ * 
+ * CARA KERJA:
+ * 1. Clone dari Gitea (untuk dapat history 75 commits)
+ * 2. Salin file proyek terbaru ke folder clone
+ * 3. Commit perubahan
+ * 4. Push ke Gitea
+ * 5. Push ke GitHub dengan remote tambahan
+ */
+
+const git     = require('isomorphic-git');
+const http    = require('isomorphic-git/http/node');
+const fs      = require('fs');
+const path    = require('path');
+const os      = require('os');
+
+// ─── CONFIG ────────────────────────────────────────────────────────────────
+const PROJECT_DIR = path.resolve(__dirname);
+
+// Gitea
+const GITEA_URL     = 'https://git.proit.id/rafif/Deep-Package-Inspection';
+const GITEA_BRANCH  = 'Proxy_Server_API_backone.cloud';
+const GITEA_USER    = 'rafif';
+const GITEA_PASS    = 'NetWorking.0';
+
+// GitHub
+const GITHUB_URL    = 'https://github.com/Rafif-Riqullah-Siregar/BackOne-Deep-Package-Inspection';
+const GITHUB_BRANCH = 'Proxy-Server-API-backone.cloud';
+// GitHub token (diisi nanti, atau bisa kosong dulu untuk test)
+const GITHUB_TOKEN  = process.env.GITHUB_TOKEN || '';
+
+// Commit message
+const COMMIT_MSG = `feat(source2): push all latest files - device labeling, help system, proxy docs, database isolation fix
+
+- Added DOKUMENTASI-FILTER-PER-SITE.md (site isolation docs)
+- Fixed start-with-env.js to force-load .env.production
+- Fixed MONGODB_URI hostname from mongodb-netify to mongodb.prod.proit.id
+- Updated .gitignore to exclude sensitive scripts and credential files
+- Minor UI and labeling improvements`;
+
+// Author
+const AUTHOR = { name: 'Rafif-Riqullah-Siregar', email: 'rafif@databisnis.id' };
+
+// ─── GITIGNORE PATTERNS ──────────────────────────────────────────────────────
+// File/folder yang TIDAK boleh di-push (dari .gitignore)
+const EXCLUDED_PATTERNS = [
+  'node_modules',
+  '.next',
+  '.env',
+  '.env.local',
+  '.env.production',
+  '.env.development',
+  'coverage',
+  'build',
+  'out',
+  '.DS_Store',
+  '*.log',
+  '*.pem',
+  '.vercel',
+  '*.tsbuildinfo',
+  'next-env.d.ts',
+  '*.db', '*.db-shm', '*.db-wal', '*.sqlite',
+  'Laporan_*.docx',
+  'temp_docx',
+  '*.zip',
+  'AGENTS.md', 'CLAUDE.md', '.agents',
+  'docs', 'plans',
+  'temp.json', 'scratch',
+  // Sensitive scripts
+  'compare-netify-vs-dashboard.js',
+  'ssh-read-source1-proxy.js',
+  'check-frontend-uri-now.js',
+  'verify-final.js',
+  'check-frontend-uri.js',
+  'ssh-check-logs.js',
+  // Sensitive docs
+  'BUKTI-AKSES-MONGODB.txt',
+  'DOKUMENTASI-PROXY-NETIFY.md',
+];
+
+function shouldExclude(filePath) {
+  const parts = filePath.split(/[/\\]/);
+  for (const pattern of EXCLUDED_PATTERNS) {
+    for (const part of parts) {
+      if (pattern.startsWith('*')) {
+        const ext = pattern.slice(1);
+        if (part.endsWith(ext)) return true;
+      } else if (part === pattern || filePath.includes(pattern)) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+async function getGitFiles(dir, baseDir = dir) {
+  const files = [];
+  const entries = fs.readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    const relPath  = path.relative(baseDir, fullPath).replace(/\\/g, '/');
+    if (shouldExclude(relPath) || entry.name === '.git') continue;
+    if (entry.isDirectory()) {
+      files.push(...await getGitFiles(fullPath, baseDir));
+    } else {
+      files.push(relPath);
+    }
+  }
+  return files;
+}
+
+async function main() {
+  console.log('╔══════════════════════════════════════════════════════════════╗');
+  console.log('║  GIT PUSH (isomorphic-git) → Gitea + GitHub                ║');
+  console.log('╚══════════════════════════════════════════════════════════════╝\n');
+
+  // ─── STEP 1: Clone dari Gitea ke temp dir ──────────────────────────────────
+  const tmpDir = path.join(os.tmpdir(), `dpi-push-${Date.now()}`);
+  console.log(`[1] Cloning dari Gitea ke temp: ${tmpDir}`);
+  fs.mkdirSync(tmpDir, { recursive: true });
+
+  try {
+    await git.clone({
+      fs, http,
+      dir: tmpDir,
+      url: GITEA_URL,
+      ref: GITEA_BRANCH,
+      singleBranch: true,
+      depth: 10,    // ambil 10 commit terakhir saja (cukup untuk push)
+      onAuth: () => ({ username: GITEA_USER, password: GITEA_PASS }),
+      onProgress: ({ phase, loaded, total }) => {
+        if (total) process.stdout.write(`\r   ${phase}: ${loaded}/${total}   `);
+      },
+    });
+    console.log(`\n   ✅ Clone berhasil dari Gitea branch ${GITEA_BRANCH}`);
+  } catch (err) {
+    console.error(`\n   ❌ Clone dari Gitea gagal: ${err.message}`);
+    console.log('   → Coba dengan username tanpa domain (tanpa @databisnis.id)');
+    process.exit(1);
+  }
+
+  // ─── STEP 2: Salin file project terbaru ke temp dir ────────────────────────
+  console.log(`\n[2] Menyalin file terbaru dari project ke clone...`);
+  const projectFiles = await getGitFiles(PROJECT_DIR);
+  let copied = 0;
+  for (const relPath of projectFiles) {
+    const src = path.join(PROJECT_DIR, relPath);
+    const dst = path.join(tmpDir, relPath);
+    fs.mkdirSync(path.dirname(dst), { recursive: true });
+    fs.copyFileSync(src, dst);
+    copied++;
+  }
+  console.log(`   ✅ ${copied} file disalin ke clone`);
+
+  // ─── STEP 3: Stage semua perubahan ─────────────────────────────────────────
+  console.log(`\n[3] Staging semua perubahan...`);
+  const statusMatrix = await git.statusMatrix({ fs, dir: tmpDir });
+  let staged = 0;
+  for (const [filepath, head, workdir, stage] of statusMatrix) {
+    if (workdir !== stage) {
+      if (workdir === 0) {
+        // File dihapus
+        await git.remove({ fs, dir: tmpDir, filepath });
+      } else {
+        // File baru atau dimodifikasi
+        await git.add({ fs, dir: tmpDir, filepath });
+      }
+      staged++;
+    }
+  }
+  console.log(`   ✅ ${staged} file di-stage`);
+
+  if (staged === 0) {
+    console.log('   ℹ️  Tidak ada perubahan yang perlu di-commit!');
+    return cleanup(tmpDir);
+  }
+
+  // ─── STEP 4: Commit ─────────────────────────────────────────────────────────
+  console.log(`\n[4] Membuat commit...`);
+  const sha = await git.commit({
+    fs,
+    dir: tmpDir,
+    author: AUTHOR,
+    committer: AUTHOR,
+    message: COMMIT_MSG,
+  });
+  console.log(`   ✅ Commit dibuat: ${sha.slice(0, 8)}`);
+
+  // ─── STEP 5: Push ke Gitea ──────────────────────────────────────────────────
+  console.log(`\n[5] Push ke Gitea (${GITEA_URL})...`);
+  try {
+    await git.push({
+      fs, http,
+      dir: tmpDir,
+      remote: 'origin',
+      ref: GITEA_BRANCH,
+      onAuth: () => ({ username: GITEA_USER, password: GITEA_PASS }),
+      onProgress: ({ phase, loaded, total }) => {
+        if (total) process.stdout.write(`\r   ${phase}: ${loaded}/${total}   `);
+      },
+    });
+    console.log(`\n   ✅ Push ke Gitea BERHASIL! Branch: ${GITEA_BRANCH}`);
+  } catch (err) {
+    console.error(`\n   ❌ Push ke Gitea gagal: ${err.message}`);
+  }
+
+  // ─── STEP 6: Push ke GitHub ─────────────────────────────────────────────────
+  console.log(`\n[6] Push ke GitHub (${GITHUB_URL})...`);
+
+  // Tambah remote GitHub
+  const remotes = await git.listRemotes({ fs, dir: tmpDir });
+  const hasGithub = remotes.some(r => r.remote === 'github');
+  if (!hasGithub) {
+    await git.addRemote({ fs, dir: tmpDir, remote: 'github', url: GITHUB_URL });
+  }
+
+  // Coba push ke GitHub
+  if (!GITHUB_TOKEN) {
+    console.log('   ⚠️  GITHUB_TOKEN tidak di-set. GitHub push membutuhkan Personal Access Token.');
+    console.log('   → Set environment variable: $env:GITHUB_TOKEN = "ghp_XXXX"');
+    console.log('   → Lalu jalankan: node git-push-iso.js');
+  } else {
+    try {
+      await git.push({
+        fs, http,
+        dir: tmpDir,
+        remote: 'github',
+        ref: GITHUB_BRANCH,
+        remoteRef: GITHUB_BRANCH,
+        onAuth: () => ({ username: 'Rafif-Riqullah-Siregar', password: GITHUB_TOKEN }),
+        onProgress: ({ phase, loaded, total }) => {
+          if (total) process.stdout.write(`\r   ${phase}: ${loaded}/${total}   `);
+        },
+      });
+      console.log(`\n   ✅ Push ke GitHub BERHASIL! Branch: ${GITHUB_BRANCH}`);
+    } catch (err) {
+      console.error(`\n   ❌ Push ke GitHub gagal: ${err.message}`);
+    }
+  }
+
+  cleanup(tmpDir);
+  console.log('\n╔══════════════════════════════════════════════════════════════╗');
+  console.log('║  SELESAI                                                    ║');
+  console.log('╚══════════════════════════════════════════════════════════════╝');
+}
+
+function cleanup(tmpDir) {
+  try {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+    console.log(`\n[cleanup] Temp dir dihapus: ${tmpDir}`);
+  } catch(e) {}
+}
+
+main().catch(err => {
+  console.error('\n[FATAL]', err.message);
+  process.exit(1);
+});

migration-temp/customagentlocations.json

@@ -0,0 +1,35 @@
+[
+  {
+    "_id": "6a584fdb36539224fa4cbfd9",
+    "agent_uuid": "F6-2V-DT-8A",
+    "site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
+    "latitude": -6.2263304,
+    "longitude": 106.4247322,
+    "label": "CPI Balaraja Agent Office",
+    "created_at": "2026-07-14T04:03:09.294Z",
+    "updated_at": "2026-07-14T04:03:09.294Z",
+    "__v": 0
+  },
+  {
+    "_id": "6a584fdb36539224fa4cbfda",
+    "agent_uuid": "2F-TF-1D-GK",
+    "site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
+    "latitude": -6.3763318,
+    "longitude": 106.8983017,
+    "label": "JRP Cibubur Agent",
+    "created_at": "2026-07-14T04:03:09.303Z",
+    "updated_at": "2026-07-14T04:57:22.288Z",
+    "__v": 0
+  },
+  {
+    "_id": "6a584fdb36539224fa4cbfdb",
+    "agent_uuid": "8A-V3-PB-85",
+    "site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
+    "latitude": -6.2253265,
+    "longitude": 106.8061484,
+    "label": "IFG LT.18 Agent HQ",
+    "created_at": "2026-07-14T04:03:09.322Z",
+    "updated_at": "2026-07-14T04:03:09.322Z",
+    "__v": 0
+  }
+]

migration-temp/tenantconfigs.json

@@ -0,0 +1,35 @@
+[
+  {
+    "_id": "6a584fdb36539224fa4cbfd6",
+    "site_uuid": "default",
+    "brand_name": "BackOne",
+    "brand_logo": "/backone-logo.png",
+    "footer_copyright": "PT. Data Bisnis Solusi",
+    "primary_color": "#E11D48",
+    "created_at": "2026-07-14T02:15:21.201Z",
+    "updated_at": "2026-07-27T01:03:28.716Z",
+    "__v": 0
+  },
+  {
+    "_id": "6a584fdb36539224fa4cbfd7",
+    "site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
+    "brand_name": "SIAB",
+    "brand_logo": "/siab-logo.png",
+    "footer_copyright": "PT. Data Bisnis Solusi",
+    "primary_color": "#3B82F6",
+    "created_at": "2026-07-14T02:15:21.204Z",
+    "updated_at": "2026-07-27T01:03:28.796Z",
+    "__v": 0
+  },
+  {
+    "_id": "6a584fdb36539224fa4cbfd8",
+    "site_uuid": "d7902405_0dc2_458b_8584_ed4d24b64f24",
+    "brand_name": "Nexus",
+    "brand_logo": "/nexus-logo.png",
+    "footer_copyright": "PT. Nexus Solusi",
+    "primary_color": "#8B5CF6",
+    "created_at": "2026-07-14T02:15:21.206Z",
+    "updated_at": "2026-07-27T01:03:28.799Z",
+    "__v": 0
+  }
+]

migration-temp/users.json

@@ -0,0 +1,217 @@
+[
+  {
+    "_id": "6a509b014fa14ba76d96ed33",
+    "username": "admin",
+    "password_hash": "$2a$10$uaBO91aVN9kwWS3rhCBvmu3uV00QFzMjorwqPK/AkhsGKKaLoFJoG",
+    "account_name": "BackOne Administrator",
+    "role": "SUPER_ADMIN",
+    "site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
+    "is_active": true,
+    "created_at": "2026-07-08T06:20:27.502Z",
+    "updated_at": "2026-07-21T06:57:52.516Z",
+    "profile_picture": "profile-1784254528937-270868858.png",
+    "__v": 0,
+    "agent_uuid": null,
+    "created_by": "admin",
+    "login_attempts": 0
+  },
+  {
+    "_id": "6a509b254fa14ba76d96ed35",
+    "username": "cibubur",
+    "password_hash": "$2a$10$OjvVNyBXRogLWcBS07GHUOkBVtBMZ6iue6U71PgWWlbMXDWe9kCu.",
+    "account_name": "JRP Cibubur Agent",
+    "role": "AGENT_VIEWER",
+    "site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
+    "agent_uuid": "2F-TF-1D-GK",
+    "is_active": true,
+    "created_at": "2026-07-14T03:39:34.474Z",
+    "__v": 0,
+    "created_by": "admin",
+    "profile_picture": null,
+    "updated_at": "2026-07-17T13:57:02.823Z",
+    "login_attempts": 0
+  },
+  {
+    "_id": "6a509b2e4fa14ba76d96ed36",
+    "username": "ifg",
+    "password_hash": "$2a$10$MsoJJqgY98DmgGMGyQIUD.PRA5kdbpXWhvNHFRakeipuJGF2F/73q",
+    "account_name": "IFG LT.18 Agent",
+    "role": "AGENT_VIEWER",
+    "site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
+    "agent_uuid": "8A-V3-PB-85",
+    "is_active": true,
+    "created_at": "2026-07-14T03:39:52.757Z",
+    "__v": 0,
+    "created_by": "admin",
+    "profile_picture": null,
+    "updated_at": "2026-07-14T03:40:22.272Z"
+  },
+  {
+    "_id": "6a509b394fa14ba76d96ed37",
+    "username": "balaraja",
+    "password_hash": "$2a$10$sx7XNdylDOr1XCy4PjjEuOrCoIWhayMNYwNlsAjspHVnmrz0xmQ9a",
+    "account_name": "CPI Balaraja Agent",
+    "role": "AGENT_VIEWER",
+    "site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
+    "agent_uuid": "F6-2V-DT-8A",
+    "is_active": true,
+    "created_at": "2026-07-14T03:40:14.386Z",
+    "__v": 0,
+    "created_by": "admin",
+    "profile_picture": null,
+    "updated_at": "2026-07-14T03:40:14.386Z"
+  },
+  {
+    "_id": "6a509b424fa14ba76d96ed38",
+    "username": "007",
+    "password_hash": "$2a$10$CDc8GOc0aTQaqMm/jD8E5OvYTxr.lbTPdrRbC6O1D2MDRzClbgyA6",
+    "account_name": "Gateway 007 Agent",
+    "role": "AGENT_VIEWER",
+    "site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
+    "agent_uuid": "YW-6I-61-LL",
+    "is_active": true,
+    "created_at": "2026-07-14T03:40:51.583Z",
+    "__v": 0,
+    "created_by": "admin",
+    "profile_picture": null,
+    "updated_at": "2026-07-17T09:10:21.716Z",
+    "login_attempts": 3,
+    "lockout_until": "2026-07-17T09:25:21.716Z"
+  },
+  {
+    "_id": "6a54b314301004e28818f8e2",
+    "username": "bsd",
+    "password_hash": "$2a$10$IIZtN8coQVLfptktA0bO2eIzaCpy3yIGtr9EUWsSf9MdTUaztx8eW",
+    "account_name": "Fazza BSD",
+    "profile_picture": null,
+    "role": "AGENT_VIEWER",
+    "site_uuid": "d7902405_0dc2_458b_8584_ed4d24b64f24",
+    "agent_uuid": "1T-5Q-RC-AS",
+    "is_active": true,
+    "created_at": "2026-07-14T03:38:04.913Z",
+    "updated_at": "2026-07-14T03:38:04.913Z",
+    "__v": 0,
+    "created_by": "admin"
+  },
+  {
+    "_id": "6a54b332301004e28818f8e8",
+    "username": "jkt",
+    "password_hash": "$2a$10$EE0G6vQ6qbN6MYj2BSjqWOdJN2q/LmBcYRLZ578higbfLjnOzRKuW",
+    "account_name": "Fazza JKT",
+    "profile_picture": null,
+    "role": "AGENT_VIEWER",
+    "site_uuid": "d7902405_0dc2_458b_8584_ed4d24b64f24",
+    "agent_uuid": "2N-ID-VQ-AL",
+    "is_active": true,
+    "created_at": "2026-07-14T03:38:25.721Z",
+    "updated_at": "2026-07-14T03:38:25.721Z",
+    "__v": 0,
+    "created_by": "admin"
+  },
+  {
+    "_id": "6a56617fe7a6bc10808db750",
+    "username": "siab",
+    "__v": 0,
+    "account_name": "SIAB Administrator",
+    "agent_uuid": null,
+    "created_at": "2026-07-14T03:13:43.107Z",
+    "created_by": "admin",
+    "is_active": true,
+    "password_hash": "$2a$10$lGP.s16GBImnBWKlFCg9Ge7d0k0vwwhFGrlfExRs6KlhO/fW6yJE.",
+    "profile_picture": "profile-1784254601947-954448750.png",
+    "role": "TENANT_ADMIN",
+    "site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
+    "updated_at": "2026-07-17T02:16:41.972Z"
+  },
+  {
+    "_id": "6a56617fe7a6bc10808db751",
+    "username": "nexus",
+    "__v": 0,
+    "account_name": "Nexus Administrator",
+    "agent_uuid": null,
+    "created_at": "2026-07-14T03:23:28.022Z",
+    "created_by": "nexus",
+    "is_active": true,
+    "password_hash": "$2a$10$nwhAiFodTWGZChddy10uvOV7jjo1aNMoJqrr9yB58GqGJE9oixaSe",
+    "profile_picture": "profile-1784254553441-339825741.png",
+    "role": "TENANT_ADMIN",
+    "site_uuid": "d7902405_0dc2_458b_8584_ed4d24b64f24",
+    "updated_at": "2026-07-17T09:37:28.382Z",
+    "login_attempts": 0
+  },
+  {
+    "_id": "6a56617fe7a6bc10808db752",
+    "username": "sulist",
+    "__v": 0,
+    "account_name": "BackOne Super SOC Analyst",
+    "agent_uuid": null,
+    "created_at": "2026-07-14T03:41:18.852Z",
+    "created_by": "admin",
+    "is_active": true,
+    "password_hash": "$2a$10$jNV0a9uQrtnvNJwkHVe2b.m0NBOXFSbGN/6cku/wCdeuV9p9gpmSq",
+    "profile_picture": "profile-1784254618510-383483774.png",
+    "role": "SOC_ANALYST",
+    "site_uuid": null,
+    "updated_at": "2026-07-17T02:16:58.532Z"
+  },
+  {
+    "_id": "6a56617fe7a6bc10808db753",
+    "username": "nelis",
+    "__v": 0,
+    "account_name": "Nexus SOC Analyst",
+    "agent_uuid": null,
+    "created_at": "2026-07-14T03:42:02.608Z",
+    "created_by": "nexus",
+    "is_active": true,
+    "password_hash": "$2a$10$tKdI9yz1e9V2mSW4H/gj.udLtAtSrHJy0QxMbq6hN3mym5XxJpH.W",
+    "profile_picture": "profile-1784254567483-97901195.png",
+    "role": "SOC_ANALYST",
+    "site_uuid": "d7902405_0dc2_458b_8584_ed4d24b64f24",
+    "updated_at": "2026-07-17T02:16:07.500Z"
+  },
+  {
+    "_id": "6a56617fe7a6bc10808db754",
+    "username": "nener",
+    "__v": 0,
+    "account_name": "Nexus Engineer",
+    "agent_uuid": null,
+    "created_at": "2026-07-14T03:44:55.970Z",
+    "created_by": "nexus",
+    "is_active": true,
+    "password_hash": "$2a$10$OoaKeuEeSHkqYMy1W50tvegaQm7u3qpP1YWuBcfmyJ45aH1kwL.Oq",
+    "profile_picture": "profile-1784254581808-854860134.png",
+    "role": "ENGINEER",
+    "site_uuid": "d7902405_0dc2_458b_8584_ed4d24b64f24",
+    "updated_at": "2026-07-17T02:16:21.824Z"
+  },
+  {
+    "_id": "6a56617fe7a6bc10808db755",
+    "username": "silis",
+    "__v": 0,
+    "account_name": "SIAB SOC Analyst",
+    "agent_uuid": null,
+    "created_at": "2026-07-14T03:51:30.034Z",
+    "created_by": "siab",
+    "is_active": true,
+    "password_hash": "$2a$10$LrDCN9PzBjBV5uKKDIkcjOZcfq3WADJM408.VBrwlQmeqO/PbZtoG",
+    "profile_picture": "profile-1784254634906-830642874.png",
+    "role": "SOC_ANALYST",
+    "site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
+    "updated_at": "2026-07-17T02:17:14.925Z"
+  },
+  {
+    "_id": "6a56617fe7a6bc10808db756",
+    "username": "siner",
+    "__v": 0,
+    "account_name": "SIAB Engineer",
+    "agent_uuid": null,
+    "created_at": "2026-07-14T03:51:50.884Z",
+    "created_by": "siab",
+    "is_active": true,
+    "password_hash": "$2a$10$3CISy5oSUYnC23Whfwf36OSE3y7DHYBXDXRvJwZBldyQD0ehc5Nlm",
+    "profile_picture": "profile-1784254648483-456467829.png",
+    "role": "ENGINEER",
+    "site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
+    "updated_at": "2026-07-17T02:17:28.503Z"
+  }
+]

next.config.ts

@@ -2,17 +2,16 @@ import type { NextConfig } from "next";
 
 const nextConfig: NextConfig = {
   output: "standalone",
-  serverExternalPackages: ["mongoose"],
   experimental: {
     serverActions: {
-      allowedOrigins: ["demoplace.my.id", "www.demoplace.my.id"],
+      allowedOrigins: ["dev.demoplace.my.id", "www.dev.demoplace.my.id"],
     },
   },
   async rewrites() {
     return [
       {
         source: '/api/:path*',
-        destination: `${process.env.NEXT_PUBLIC_API_URL || 'http://127.0.0.1:3001'}/api/:path*`,
+        destination: `${process.env.NEXT_PUBLIC_API_URL || 'http://127.0.0.1:3011'}/api/:path*`,
       },
     ];
   },

nginx/conf.d/default.conf

@@ -1,8 +1,14 @@
+limit_req_zone $binary_remote_addr zone=api:10m rate=30r/s;
+limit_req_zone $binary_remote_addr zone=uploads:10m rate=5r/s;
+
 server {
     listen 80;
-    server_name demoplace.my.id;
+    server_name demoplace.my.id www.demoplace.my.id;
-    
+
-    server_tokens off; # Hide NGINX version
+    server_tokens off;
+
+    # Allow large file uploads for profile pictures (max 10MB)
+    client_max_body_size 10m;
 
     # Security Headers
     add_header X-Frame-Options "SAMEORIGIN" always;
@@ -10,12 +16,12 @@ server {
     add_header X-Content-Type-Options "nosniff" always;
     add_header Referrer-Policy "no-referrer-when-downgrade" always;
     add_header Content-Security-Policy "default-src 'self' http: https: data: blob: 'unsafe-inline' 'unsafe-eval';" always;
+    add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
 
-    # Rate Limiting zone configuration should be in nginx.conf (http block), but we can configure basic protection
+    # Rate limiting on API endpoints
-    # We will pass everything to the frontend container
+    location /api/auth/ {
-    
+        limit_req zone=api burst=20 nodelay;
-    location / {
+        proxy_pass http://127.0.0.1:3000;
-        proxy_pass http://frontend:3000;
         proxy_http_version 1.1;
         proxy_set_header Upgrade $http_upgrade;
         proxy_set_header Connection 'upgrade';
@@ -24,8 +30,37 @@ server {
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_hide_header X-Powered-By;
+        proxy_read_timeout 30s;
+    }
 
-        # Hide internal technologies from being sent back to the client
+    # Profile picture uploads — rate limited more strictly
+    location /api/auth/upload-profile-picture {
+        limit_req zone=uploads burst=5 nodelay;
+        client_max_body_size 10m;
+        proxy_pass http://127.0.0.1:3000;
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_read_timeout 60s;
         proxy_hide_header X-Powered-By;
     }
+
+    # All other traffic goes to Next.js frontend
+    location / {
+        proxy_pass http://127.0.0.1:3000;
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection 'upgrade';
+        proxy_set_header Host $host;
+        proxy_cache_bypass $http_upgrade;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_hide_header X-Powered-By;
+        proxy_read_timeout 30s;
+        proxy_connect_timeout 10s;
+    }
 }

package.json

@@ -1,13 +1,17 @@
 {
-  "name": "netify-app",
+  "name": "backone-dpi",
-  "version": "0.1.0",
+  "version": "1.0.0",
   "private": true,
+  "engines": {
+    "node": ">=18.0.0"
+  },
   "scripts": {
-    "dev": "concurrently --names \"NEXT,BACKEND,PROXY\" --prefix-colors \"cyan,green,yellow\" \"next dev\" \"node backend/server.js\" \"node proxy/index.js\"",
+    "dev": "concurrently --names \"MONGO,NEXT,BACKEND,PROXY\" --prefix-colors \"magenta,cyan,green,yellow\" \"node scripts/start-mongo.js\" \"next dev -p 3010\" \"node backend/server.js\" \"node proxy/index.js\"",
-    "dev:next": "next dev",
+    "dev:central": "concurrently --names \"NEXT,BACKEND\" --prefix-colors \"cyan,green\" \"next dev -p 3010\" \"node backend/server.js\"",
+    "dev:next": "next dev -p 3010",
     "dev:backend": "node backend/server.js",
     "dev:proxy": "node proxy/index.js",
-    "kill:ports": "powershell -Command \"@(3000,3001,4000) | ForEach-Object { $port = $_; $pids = netstat -ano | Select-String \\\":$port\\s+.+LISTENING\\\" | ForEach-Object { ($_ -split '\\s+')[-1] }; $pids | Where-Object { $_ -match '^\\d+$' } | ForEach-Object { taskkill /PID $_ /F 2>$null } }; Write-Host 'Ports 3000/3001/4000 cleared.'\"",
+    "kill:ports": "powershell -Command \"@(3010,3011,4010) | ForEach-Object { $port = $_; $pids = netstat -ano | Select-String \\\":$port\\s+.+LISTENING\\\" | ForEach-Object { ($_ -split '\\s+')[-1] }; $pids | Where-Object { $_ -match '^\\d+$' } | ForEach-Object { taskkill /PID $_ /F 2>$null } }; Write-Host 'Ports 3010/3011/4010 cleared.'\"",
     "build": "next build",
     "start": "next start",
     "start:prod": "concurrently \"next start\" \"node backend/server.js\" \"node proxy/index.js\"",
@@ -15,9 +19,12 @@
     "backend": "node backend/server.js",
     "proxy": "node proxy/index.js",
     "proxy:bun": "bun proxy/index.js",
-    "install:all": "npm install && cd backend && npm install && cd ../proxy && npm install && cd .."
+    "install:all": "npm install && cd backend && npm install && cd ../proxy && npm install && cd ..",
+    "deploy": "npm run build && node scripts/deploy-sftp.js",
+    "deploy:sftp": "node scripts/deploy-sftp.js"
   },
   "dependencies": {
+    "@react-pdf/renderer": "^4.5.1",
     "@types/leaflet": "^1.9.21",
     "axios": "^1.18.1",
     "bcryptjs": "^3.0.3",
@@ -31,6 +38,7 @@
     "dotenv": "^17.4.2",
     "express": "^5.2.1",
     "framer-motion": "^12.42.2",
+    "isomorphic-git": "^1.40.0",
     "jsonwebtoken": "^9.0.3",
     "leaflet": "^1.9.4",
     "lucide-react": "^1.21.0",
@@ -40,6 +48,7 @@
     "next": "16.2.9",
     "next-themes": "^0.4.6",
     "node-cron": "^4.6.0",
+    "node-fetch": "^3.3.2",
     "react": "19.2.4",
     "react-dom": "19.2.4",
     "react-globe.gl": "^2.38.0",

postcss.config.mjs

@@ -1,7 +1,7 @@
-const config = {
+const config = {
-  plugins: {
+  plugins: {
-    "@tailwindcss/postcss": {},
+    "@tailwindcss/postcss": {},
-  },
+  },
-};
+};
-
+
-export default config;
+export default config;

proxy/Dockerfile.bun

@@ -1,20 +1,20 @@
-FROM oven/bun:1-alpine
+FROM oven/bun:1-alpine
-
+
-WORKDIR /app
+WORKDIR /app
-
+
-# Install dependencies first (layer caching)
+# Install dependencies first (layer caching)
-# bun install is compatible with npm package.json / package-lock.json
+# bun install is compatible with npm package.json / package-lock.json
-COPY package*.json ./
+COPY package*.json ./
-RUN bun install --production
+RUN bun install --production
-
+
-# Copy application source
+# Copy application source
-COPY . .
+COPY . .
-
+
-# Expose proxy REST API port
+# Expose proxy REST API port
-EXPOSE 4000
+EXPOSE 4000
-
+
-# Health check
+# Health check
-HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \
+HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \
-  CMD bun -e "require('http').get('http://localhost:4000/health', r => r.statusCode === 200 ? process.exit(0) : process.exit(1)).on('error', () => process.exit(1))"
+  CMD bun -e "require('http').get('http://localhost:4000/health', r => r.statusCode === 200 ? process.exit(0) : process.exit(1)).on('error', () => process.exit(1))"
-
+
-CMD ["bun", "run", "index.js"]
+CMD ["bun", "run", "index.js"]

proxy/INFO.md

@@ -1,108 +1,108 @@
-# BackOne DPI Proxy — Deployment Reference
+# BackOne DPI Proxy — Deployment Reference
-
+
-Standalone Docker image for collecting Netify DPI data and writing to MongoDB.
+Standalone Docker image for collecting Netify DPI data and writing to MongoDB.
-
+
----
+---
-
+
-## Environment Variables
+## Environment Variables
-
+
-### Required
+### Required
-
+
-| Variable | Description |
+| Variable | Description |
-|---|---|
+|---|---|
-| `NETIFY_SITE_UUIDS` | Comma-separated Netify site UUIDs (or single `NETIFY_SITE_UUID`) |
+| `NETIFY_SITE_UUIDS` | Comma-separated Netify site UUIDs (or single `NETIFY_SITE_UUID`) |
-| `NETIFY_TOKEN` | Netify JWT token (or `NETIFY_JWT_TOKEN` / `NETIFY_API_KEY`) |
+| `NETIFY_TOKEN` | Netify JWT token (or `NETIFY_JWT_TOKEN` / `NETIFY_API_KEY`) |
-
+
-### MongoDB
+### MongoDB
-
+
-| Variable | Default | Description |
+| Variable | Default | Description |
-|---|---|---|
+|---|---|---|
-| `MONGODB_URI` | `mongodb://127.0.0.1:27017/backone_dpi` | MongoDB connection string |
+| `MONGODB_URI` | `mongodb://127.0.0.1:27017/backone_dpi` | MongoDB connection string |
-
+
-### Collection Mode
+### Collection Mode
-
+
-| Variable | Default | Description |
+| Variable | Default | Description |
-|---|---|---|
+|---|---|---|
-| `PROXY_COLLECT_MODE` | `all` | `all` = all agents, `agent` = single agent, `agents` = list of agents |
+| `PROXY_COLLECT_MODE` | `all` | `all` = all agents, `agent` = single agent, `agents` = list of agents |
-| `PROXY_AGENT_UUID` | _(none)_ | Single agent UUID (required if `mode=agent`) |
+| `PROXY_AGENT_UUID` | _(none)_ | Single agent UUID (required if `mode=agent`) |
-| `PROXY_AGENT_UUIDS` | _(none)_ | Comma-separated agent UUIDs (required if `mode=agents`) |
+| `PROXY_AGENT_UUIDS` | _(none)_ | Comma-separated agent UUIDs (required if `mode=agents`) |
-| `PROXY_AGENT_DELAY_MS` | `5000` | Delay (ms) between each agent collection to avoid rate-limiting |
+| `PROXY_AGENT_DELAY_MS` | `5000` | Delay (ms) between each agent collection to avoid rate-limiting |
-
+
-### Scheduling
+### Scheduling
-
+
-| Variable | Default | Description |
+| Variable | Default | Description |
-|---|---|---|
+|---|---|---|
-| `PROXY_CRON_SCHEDULE` | `*/5 * * * *` | Cron expression for collection interval |
+| `PROXY_CRON_SCHEDULE` | `*/5 * * * *` | Cron expression for collection interval |
-| `PROXY_CAPACITY_LOG_INTERVAL_MS` | `86400000` | How often to log DB capacity usage (default: 24h) |
+| `PROXY_CAPACITY_LOG_INTERVAL_MS` | `86400000` | How often to log DB capacity usage (default: 24h) |
-
+
-### Limits
+### Limits
-
+
-| Variable | Default | Description |
+| Variable | Default | Description |
-|---|---|---|
+|---|---|---|
-| `PROXY_FLOW_LIMIT` | `1000000` | Max flows to fetch per agent per cycle |
+| `PROXY_FLOW_LIMIT` | `1000000` | Max flows to fetch per agent per cycle |
-| `PROXY_PORT` | `4000` | REST API listen port |
+| `PROXY_PORT` | `4000` | REST API listen port |
-
+
-### Netify API
+### Netify API
-
+
-| Variable | Default | Description |
+| Variable | Default | Description |
-|---|---|---|
+|---|---|---|
-| `NETIFY_INFORMATICS_BASE_URL` | `https://informatics.netify.ai/api/v1` | Netify API base URL |
+| `NETIFY_INFORMATICS_BASE_URL` | `https://informatics.netify.ai/api/v1` | Netify API base URL |
-
+
----
+---
-
+
-## Docker Run Example
+## Docker Run Example
-
+
-```bash
+```bash
-docker run -d \
+docker run -d \
-  --name backone_proxy \
+  --name backone_proxy \
-  -p 4000:4000 \
+  -p 4000:4000 \
-  -e MONGODB_URI=mongodb://host.docker.internal:27017/backone_dpi \
+  -e MONGODB_URI=mongodb://host.docker.internal:27017/backone_dpi \
-  -e NETIFY_SITE_UUIDS=site-uuid-1,site-uuid-2 \
+  -e NETIFY_SITE_UUIDS=site-uuid-1,site-uuid-2 \
-  -e NETIFY_TOKEN=your-jwt-token \
+  -e NETIFY_TOKEN=your-jwt-token \
-  -e PROXY_COLLECT_MODE=agents \
+  -e PROXY_COLLECT_MODE=agents \
-  -e PROXY_AGENT_UUIDS=agent-uuid-1,agent-uuid-2,agent-uuid-3 \
+  -e PROXY_AGENT_UUIDS=agent-uuid-1,agent-uuid-2,agent-uuid-3 \
-  backone-proxy
+  backone-proxy
-```
+```
-
+
-## Docker Compose Example
+## Docker Compose Example
-
+
-```yaml
+```yaml
-services:
+services:
-  proxy:
+  proxy:
-    build: ./proxy
+    build: ./proxy
-    container_name: backone_proxy
+    container_name: backone_proxy
-    restart: always
+    restart: always
-    ports:
+    ports:
-      - "4000:4000"
+      - "4000:4000"
-    environment:
+    environment:
-      - MONGODB_URI=mongodb://mongodb:27017/backone_dpi
+      - MONGODB_URI=mongodb://mongodb:27017/backone_dpi
-      - PROXY_PORT=4000
+      - PROXY_PORT=4000
-      - PROXY_COLLECT_MODE=all
+      - PROXY_COLLECT_MODE=all
-      - PROXY_COLLECT_MODE=${PROXY_COLLECT_MODE:-all}
+      - PROXY_COLLECT_MODE=${PROXY_COLLECT_MODE:-all}
-      - PROXY_AGENT_UUID=${PROXY_AGENT_UUID:-}
+      - PROXY_AGENT_UUID=${PROXY_AGENT_UUID:-}
-      - PROXY_AGENT_UUIDS=${PROXY_AGENT_UUIDS:-}
+      - PROXY_AGENT_UUIDS=${PROXY_AGENT_UUIDS:-}
-      - PROXY_AGENT_DELAY_MS=${PROXY_AGENT_DELAY_MS:-5000}
+      - PROXY_AGENT_DELAY_MS=${PROXY_AGENT_DELAY_MS:-5000}
-      - PROXY_CRON_SCHEDULE=${PROXY_CRON_SCHEDULE:-*/5 * * * *}
+      - PROXY_CRON_SCHEDULE=${PROXY_CRON_SCHEDULE:-*/5 * * * *}
-      - NETIFY_SITE_UUIDS=${NETIFY_SITE_UUIDS}
+      - NETIFY_SITE_UUIDS=${NETIFY_SITE_UUIDS}
-      - NETIFY_TOKEN=${NETIFY_TOKEN}
+      - NETIFY_TOKEN=${NETIFY_TOKEN}
-      - NETIFY_INFORMATICS_BASE_URL=${NETIFY_INFORMATICS_BASE_URL:-https://informatics.netify.ai/api/v1}
+      - NETIFY_INFORMATICS_BASE_URL=${NETIFY_INFORMATICS_BASE_URL:-https://informatics.netify.ai/api/v1}
-```
+```
-
+
-## REST API Endpoints
+## REST API Endpoints
-
+
-| Method | Endpoint | Description |
+| Method | Endpoint | Description |
-|---|---|---|
+|---|---|---|
-| `GET` | `/health` | Liveness check (MongoDB status) |
+| `GET` | `/health` | Liveness check (MongoDB status) |
-| `GET` | `/status` | Scheduler status, mode, last run |
+| `GET` | `/status` | Scheduler status, mode, last run |
-| `GET` | `/agents` | List agent UUIDs in MongoDB |
+| `GET` | `/agents` | List agent UUIDs in MongoDB |
-| `POST` | `/collect/all` | Manual trigger — all agents |
+| `POST` | `/collect/all` | Manual trigger — all agents |
-| `POST` | `/collect/:uuid` | Manual trigger — single agent |
+| `POST` | `/collect/:uuid` | Manual trigger — single agent |
-| `POST` | `/collect/agents` | Manual trigger — multiple agents `{"uuids":[...], "delay_ms":5000}` |
+| `POST` | `/collect/agents` | Manual trigger — multiple agents `{"uuids":[...], "delay_ms":5000}` |
-| `GET` | `/latest` | Latest data from all collections (debug) |
+| `GET` | `/latest` | Latest data from all collections (debug) |
-| `GET` | `/domain-details?domain=...` | IP/MAC details for a domain |
+| `GET` | `/domain-details?domain=...` | IP/MAC details for a domain |
-
+
-## Health Check
+## Health Check
-
+
-```bash
+```bash
-curl http://localhost:4000/health
+curl http://localhost:4000/health
-```
+```

proxy/check_device.js

@@ -0,0 +1,34 @@
+// check_device.js - Detailed check of 10.6.10.44 records
+const path = require('path');
+require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
+const mongoose = require('mongoose');
+
+async function run() {
+  await mongoose.connect(process.env.MONGODB_URI || 'mongodb://localhost:27017/backone');
+  const db = mongoose.connection.db;
+  
+  // Get all records for 10.6.10.44
+  const docs = await db.collection('devicestats')
+    .find({ ip_address: '10.6.10.44' })
+    .sort({ timestamp: 1 })
+    .toArray();
+  
+  console.log(`Total docs for 10.6.10.44: ${docs.length}`);
+  docs.forEach((d, i) => {
+    console.log(`\n--- Doc ${i + 1} ---`);
+    console.log('  _id:        ', d._id);
+    console.log('  agent_uuid: ', d.agent_uuid);
+    console.log('  timestamp:  ', d.timestamp);
+    console.log('  created_at: ', d.created_at);
+    console.log('  updated_at: ', d.updated_at);
+    console.log('  download:   ', d.download);
+    console.log('  device_label:', d.device_label);
+  });
+  
+  // Check if there are different agent_uuids
+  const agents = [...new Set(docs.map(d => d.agent_uuid))];
+  console.log('\nDistinct agent_uuids for this IP:', agents);
+  
+  await mongoose.disconnect();
+}
+run().catch(err => { console.error(err.message); process.exit(1); });

proxy/check_summary_data.js

@@ -0,0 +1,63 @@
+const path = require('path');
+require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
+const mongoose = require('mongoose');
+
+const MONGODB_URI = process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017/backone_dpi';
+const SIAB = '6681452d_9cae_4ff4_8ae8_0d504774265e';
+
+mongoose.connect(MONGODB_URI).then(async () => {
+  const db = mongoose.connection.db;
+  const since24h = new Date(Date.now() - 24 * 3600000);
+  const since7d   = new Date(Date.now() - 7  * 24 * 3600000);
+
+  // Count site-level summary docs
+  const count24h = await db.collection('summaries').countDocuments({
+    site_uuid: SIAB, agent_uuid: null, timestamp: { $gte: since24h }
+  });
+  const countAll = await db.collection('summaries').countDocuments({
+    site_uuid: SIAB, agent_uuid: null
+  });
+
+  // Sum bandwidth for last 24h (site-level, agent_uuid: null)
+  const agg24h = await db.collection('summaries').aggregate([
+    { $match: { site_uuid: SIAB, agent_uuid: null, timestamp: { $gte: since24h } } },
+    { $group: { _id: null, totalDown: { $sum: '$bandwidth_down' }, totalUp: { $sum: '$bandwidth_up' }, count: { $sum: 1 } } }
+  ]).toArray();
+
+  // Sum bandwidth ALL time (site-level)
+  const aggAll = await db.collection('summaries').aggregate([
+    { $match: { site_uuid: SIAB, agent_uuid: null } },
+    { $group: { _id: null, totalDown: { $sum: '$bandwidth_down' }, totalUp: { $sum: '$bandwidth_up' }, count: { $sum: 1 } } }
+  ]).toArray();
+
+  // Oldest and newest
+  const oldest = await db.collection('summaries').findOne({ site_uuid: SIAB, agent_uuid: null }, { sort: { timestamp: 1 }, projection: { timestamp: 1 } });
+  const newest = await db.collection('summaries').findOne({ site_uuid: SIAB, agent_uuid: null }, { sort: { timestamp: -1 }, projection: { timestamp: 1, bandwidth_down: 1, bandwidth_up: 1 } });
+
+  console.log('\n=== MongoDB Summary Check (SIAB site) ===');
+  console.log('Total site-level docs:', countAll);
+  console.log('Site-level docs in last 24h:', count24h);
+  console.log('\nBandwidth SUM (last 24h):');
+  console.log('  Down:', agg24h[0] ? (agg24h[0].totalDown / 1024 / 1024).toFixed(2) + ' MB' : '0 MB');
+  console.log('  Up  :', agg24h[0] ? (agg24h[0].totalUp   / 1024 / 1024).toFixed(2) + ' MB' : '0 MB');
+  console.log('\nBandwidth SUM (ALL time):');
+  console.log('  Down:', aggAll[0] ? (aggAll[0].totalDown / 1024 / 1024).toFixed(2) + ' MB' : '0 MB');
+  console.log('  Up  :', aggAll[0] ? (aggAll[0].totalUp   / 1024 / 1024).toFixed(2) + ' MB' : '0 MB');
+  console.log('\nOldest entry :', oldest?.timestamp);
+  console.log('Newest entry :', newest?.timestamp);
+  console.log('Latest bandwidth_down per 5min:', newest ? (newest.bandwidth_down / 1024 / 1024).toFixed(4) + ' MB' : 'N/A');
+  console.log('Latest bandwidth_up per 5min  :', newest ? (newest.bandwidth_up   / 1024 / 1024).toFixed(4) + ' MB' : 'N/A');
+
+  // Also check flow data for comparison
+  const flowAgg = await db.collection('flows').aggregate([
+    { $match: { site_uuid: SIAB, timestamp: { $gte: since24h } } },
+    { $group: { _id: null, totalDown: { $sum: '$download' }, totalUp: { $sum: '$upload' }, count: { $sum: 1 } } }
+  ]).toArray();
+  console.log('\nFlow-level bandwidth (last 24h from flows collection):');
+  console.log('  Down:', flowAgg[0] ? (flowAgg[0].totalDown / 1024 / 1024).toFixed(2) + ' MB' : '0 MB');
+  console.log('  Up  :', flowAgg[0] ? (flowAgg[0].totalUp   / 1024 / 1024).toFixed(2) + ' MB' : '0 MB');
+  console.log('  Flow count:', flowAgg[0]?.count || 0);
+  console.log('=====================================\n');
+
+  process.exit(0);
+}).catch(e => { console.error(e.message); process.exit(1); });

proxy/clean_devicestat_duplicates.js

@@ -0,0 +1,73 @@
+// proxy/clean_devicestat_duplicates.js
+// ─────────────────────────────────────────────────────────────────────────────
+// One-time cleanup script to deduplicate historical DeviceStat records.
+// Keeps only the LATEST document per (agent_uuid, ip_address) pair,
+// removing all older duplicates accumulated before the upsert fix.
+//
+// Usage: node proxy/clean_devicestat_duplicates.js
+// ─────────────────────────────────────────────────────────────────────────────
+
+const path = require('path');
+require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
+
+const mongoose = require('mongoose');
+const MONGODB_URI = process.env.MONGODB_URI || 'mongodb://localhost:27017/backone';
+
+const DeviceStatSchema = new mongoose.Schema({
+  timestamp:    { type: Date },
+  agent_uuid:   { type: String },
+  site_uuid:    { type: String },
+  ip_address:   { type: String },
+  mac_address:  { type: String },
+  device_label: String,
+  device_type:  String,
+  os_label:     String,
+  manufacturer: String,
+  download:     Number,
+  upload:       Number,
+  flows:        Number,
+  last_seen:    String,
+}, { timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' } });
+
+const DeviceStat = mongoose.model('DeviceStat', DeviceStatSchema);
+
+async function run() {
+  console.log('[Cleanup] Connecting to MongoDB...');
+  await mongoose.connect(MONGODB_URI);
+  console.log('[Cleanup] Connected.');
+
+  // Find all unique (agent_uuid, ip_address) combinations
+  const groups = await DeviceStat.aggregate([
+    { $group: {
+        _id:        { agent_uuid: '$agent_uuid', ip_address: '$ip_address' },
+        ids:        { $push: '$_id' },
+        timestamps: { $push: '$timestamp' },
+        count:      { $sum: 1 },
+    }},
+    { $match: { count: { $gt: 1 } } },
+  ]);
+
+  console.log(`[Cleanup] Found ${groups.length} (agent_uuid, ip_address) pairs with duplicates.`);
+  let totalDeleted = 0;
+
+  for (const group of groups) {
+    // Sort the ids by matching timestamps - keep the latest
+    const paired = group.ids.map((id, i) => ({ id, ts: group.timestamps[i] }));
+    paired.sort((a, b) => new Date(b.ts) - new Date(a.ts));
+
+    // Keep the first (newest), delete the rest
+    const toDelete = paired.slice(1).map(p => p.id);
+    const result = await DeviceStat.deleteMany({ _id: { $in: toDelete } });
+    totalDeleted += result.deletedCount;
+  }
+
+  const remaining = await DeviceStat.countDocuments();
+  console.log(`[Cleanup] Done. Deleted ${totalDeleted} duplicate DeviceStat records.`);
+  console.log(`[Cleanup] Remaining DeviceStat documents: ${remaining}`);
+  await mongoose.disconnect();
+}
+
+run().catch(err => {
+  console.error('[Cleanup] Fatal error:', err.message);
+  process.exit(1);
+});

proxy/cleanup_duplicate_agents.js

@@ -8,8 +8,8 @@ const mongoose = require('mongoose');
 const path = require('path');
 require('dotenv').config({ path: path.join(__dirname, '../../../..', '.env.local') });
 
-const SIAB_UUID  = '6681452d_9cae_4ff4_8ae8_0d504774265e';
+const SIAB_UUID   = '6681452d_9cae_4ff4_8ae8_0d504774265e';
-const NEXUS_UUID = 'd7902405_0dc2_458b_8584_ed4d24b64f24';
+const OFFICE_UUID = '1959bb55_045b_47c7_bbdd_f33b7db197b9';
 
 // Definitive SIAB agent list (from most recent collector run)
 const SIAB_AGENTS = ['F6-2V-DT-8A', 'YW-6I-61-LL', '2F-TF-1D-GK', '1R-79-J9-YE', '8A-V3-PB-85'];
@@ -27,13 +27,13 @@ async function cleanup() {
   for (const colName of collections) {
     const col = db.collection(colName);
 
-    // 1. Delete SIAB agents that are stored under NEXUS site_uuid
+    // 1. Delete SIAB agents that are stored under OFFICE site_uuid
     const r1 = await col.deleteMany({
-      site_uuid: NEXUS_UUID,
+      site_uuid: OFFICE_UUID,
       agent_uuid: { $in: SIAB_AGENTS }
     });
     if (r1.deletedCount > 0) {
-      console.log(`[${colName}] Removed ${r1.deletedCount} docs (SIAB agents from NEXUS)`);
+      console.log(`[${colName}] Removed ${r1.deletedCount} docs (SIAB agents from OFFICE)`);
       totalDeleted += r1.deletedCount;
     }
 

proxy/collector.js

@@ -1,262 +1,263 @@
-// proxy/collector.js
+// proxy/collector.js
-// ─────────────────────────────────────────────────────────────────────────────
+// ─────────────────────────────────────────────────────────────────────────────
-// Core data collection logic for the BackOne Proxy Server
+// Core data collection logic for the BackOne Proxy Server
-// Supports 2 modes: ALL Agents and ONE Agent by UUID
+// Supports 2 modes: ALL Agents and ONE Agent by UUID
-// All data is stored in MongoDB, tagged with agent_uuid + site_uuid.
+// All data is stored in MongoDB, tagged with agent_uuid + site_uuid.
-// ─────────────────────────────────────────────────────────────────────────────
+// ─────────────────────────────────────────────────────────────────────────────
-
+
-const path = require('path');
+const path = require('path');
-require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
+require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
-
+
-const netify = require('./netifyClient');
+const netify = require('./netifyClient');
-const { collectSecondaryTelemetry } = require('./collectorHelper');
+const { collectSecondaryTelemetry } = require('./collectorHelper');
-const {
+const { collectDevicesAndApps, collectFlows } = require('./collectorHelperDpi2');
-  collectDevicesAndApps,
+const { collectThreats, collectEvents } = require('./collectorHelperDpi3');
-  collectFlows,
+
-  collectThreats,
+const { Summary, AppStat, AgentRegistry } = require('./models/Schemas');
-  collectEvents
+const { pruneOldData } = require('./dataRetention');
-} = require('./collectorHelperDpi2');
+
-
+const SITE_UUIDS_STR = process.env.NETIFY_SITE_UUIDS || process.env.NETIFY_SITE_UUID;
-const { Summary, AppStat } = require('./models/Schemas');
+const SITE_UUIDS = SITE_UUIDS_STR ? SITE_UUIDS_STR.split(',').map(s => s.trim()).filter(Boolean) : [];
-const { pruneOldData } = require('./dataRetention');
+
-
+async function collectForAgent(agentUuid, timestamp, siteUuid) {
-const SITE_UUIDS_STR = process.env.NETIFY_SITE_UUIDS || process.env.NETIFY_SITE_UUID;
+  const label = agentUuid || 'GLOBAL';
-const SITE_UUIDS = SITE_UUIDS_STR ? SITE_UUIDS_STR.split(',').map(s => s.trim()).filter(Boolean) : [];
+  console.log(`[Collector] → Fetching data for Agent: ${label}`);
-
+
-async function collectForAgent(agentUuid, timestamp, siteUuid) {
+  try {
-  const label = agentUuid || 'GLOBAL';
+    // 1. Summary / Bandwidth (including derived speed, packet_drops, and peak_flow_rate)
-  console.log(`[Collector] → Fetching data for Agent: ${label}`);
+    const summary = await netify.fetchBandwidthSummary(5, agentUuid, siteUuid);
-
+    if (summary) {
-  try {
+      let download_speed = 0;
-    // 1. Summary / Bandwidth (including derived speed, packet_drops, and peak_flow_rate)
+      let upload_speed = 0;
-    const summary = await netify.fetchBandwidthSummary(1440, agentUuid, siteUuid);
+      let packet_drops = 0;
-    if (summary) {
+      let peak_flow_rate = summary.active_flows || 0;
-      let download_speed = 0;
+
-      let upload_speed = 0;
+      try {
-      let packet_drops = 0;
+        const prev = await Summary.findOne({ agent_uuid: agentUuid }).sort({ timestamp: -1 }).lean();
-      let peak_flow_rate = summary.active_flows || 0;
+        const timeDiffSec = prev && prev.timestamp ? (timestamp.getTime() - new Date(prev.timestamp).getTime()) / 1000 : 300;
-
+        const activeTimeDiff = timeDiffSec > 0 ? timeDiffSec : 300;
-      try {
+        download_speed = summary.bandwidth_down / activeTimeDiff;
-        const prev = await Summary.findOne({ agent_uuid: agentUuid }).sort({ timestamp: -1 }).lean();
+        upload_speed   = summary.bandwidth_up / activeTimeDiff;
-        if (prev && prev.timestamp) {
+      } catch (err) {
-          const timeDiffSec = (timestamp.getTime() - new Date(prev.timestamp).getTime()) / 1000;
+        console.error('[Collector] Error calculating summary speeds:', err.message);
-          if (timeDiffSec > 0) {
+      }
-            const bytesDiffDown = Math.max(0, summary.bandwidth_down - (prev.bandwidth_down || 0));
+
-            const bytesDiffUp   = Math.max(0, summary.bandwidth_up - (prev.bandwidth_up || 0));
+      packet_drops = Math.floor((summary.active_flows || 0) * 0.015);
-            download_speed = bytesDiffDown / timeDiffSec;
+      peak_flow_rate = Math.floor((summary.active_flows || 0) * 1.18);
-            upload_speed   = bytesDiffUp / timeDiffSec;
+
-          }
+      const activeFlows = summary.active_flows || 0;
-        }
+      const totalBandwidth = (summary.bandwidth_down || 0) + (summary.bandwidth_up || 0);
-      } catch (err) {
+
-        console.error('[Collector] Error calculating summary speeds:', err.message);
+      const cpu_usage = Math.min(98, Math.max(1.2, parseFloat((2.5 + (activeFlows * 0.04) + (totalBandwidth / 10000000)).toFixed(2))));
-      }
+      const memory_usage = Math.min(99, Math.max(10.5, parseFloat((15.4 + (activeFlows * 0.02) + (totalBandwidth / 25000000)).toFixed(2))));
-
+      const queue_depth = Math.max(0, Math.floor((activeFlows * 0.15) + (totalBandwidth / 5000000)));
-      packet_drops = Math.floor((summary.active_flows || 0) * 0.015);
+
-      peak_flow_rate = Math.floor((summary.active_flows || 0) * 1.18);
+      await new Summary({
-
+        timestamp,
-      const activeFlows = summary.active_flows || 0;
+        agent_uuid: agentUuid,
-      const totalBandwidth = (summary.bandwidth_down || 0) + (summary.bandwidth_up || 0);
+        site_uuid: siteUuid,
-
+        ...summary,
-      const cpu_usage = Math.min(98, Math.max(1.2, parseFloat((2.5 + (activeFlows * 0.04) + (totalBandwidth / 10000000)).toFixed(2))));
+        download_speed,
-      const memory_usage = Math.min(99, Math.max(10.5, parseFloat((15.4 + (activeFlows * 0.02) + (totalBandwidth / 25000000)).toFixed(2))));
+        upload_speed,
-      const queue_depth = Math.max(0, Math.floor((activeFlows * 0.15) + (totalBandwidth / 5000000)));
+        packet_drops,
-
+        peak_flow_rate,
-      await new Summary({
+        cpu_usage,
-        timestamp,
+        memory_usage,
-        agent_uuid: agentUuid,
+        queue_depth
-        site_uuid: siteUuid,
+      }).save();
-        ...summary,
+      console.log(`[Collector]   ✓ Summary saved for ${label}`);
-        download_speed,
+    }
-        upload_speed,
+
-        packet_drops,
+    // 2. Top Apps
-        peak_flow_rate,
+    const apps = await netify.fetchTopApps(5, 200, agentUuid, siteUuid);
-        cpu_usage,
+    if (apps && apps.length > 0) {
-        memory_usage,
+      const appDocs = apps.map(app => ({
-        queue_depth
+        timestamp, agent_uuid: agentUuid, site_uuid: siteUuid,
-      }).save();
+        app_label: app.application?.label || 'Unknown',
-      console.log(`[Collector]   ✓ Summary saved for ${label}`);
+        download: app.download || 0, upload: app.upload || 0, flows: app.flows || 0,
-    }
+      }));
-
+      await AppStat.insertMany(appDocs);
-    // 2. Top Apps
+      console.log(`[Collector]   ✓ ${appDocs.length} apps saved for ${label}`);
-    const apps = await netify.fetchTopApps(1440, 200, agentUuid, siteUuid);
+    }
-    if (apps && apps.length > 0) {
+
-      const appDocs = apps.map(app => ({
+    // Collect Secondary Telemetry (categories, TLS, countries, DHCP, User Agents, BitTorrent)
-        timestamp, agent_uuid: agentUuid, site_uuid: siteUuid,
+    await collectSecondaryTelemetry(agentUuid, timestamp, siteUuid, netify, label);
-        app_label: app.application?.label || 'Unknown',
+
-        download: app.download || 0, upload: app.upload || 0, flows: app.flows || 0,
+    // 2. Devices & App Records
-      }));
+    const ipToMacMap = await collectDevicesAndApps(agentUuid, timestamp, siteUuid, netify, label);
-      await AppStat.insertMany(appDocs);
+
-      console.log(`[Collector]   ✓ ${appDocs.length} apps saved for ${label}`);
+    // 3. Flows
-    }
+    await collectFlows(agentUuid, timestamp, siteUuid, netify, label, ipToMacMap);
-
+
-    // Collect Secondary Telemetry (categories, TLS, countries, DHCP, User Agents, BitTorrent)
+    // 5. Threats
-    await collectSecondaryTelemetry(agentUuid, timestamp, siteUuid, netify, label);
+    await collectThreats(agentUuid, timestamp, siteUuid, netify, label);
-
+
-    // 2. Devices & App Records
+    // 6. Events
-    const ipToMacMap = await collectDevicesAndApps(agentUuid, timestamp, siteUuid, netify, label);
+    await collectEvents(agentUuid, timestamp, siteUuid, netify, label);
-
+
-    // 3. Flows
+    return { success: true, agent_uuid: agentUuid };
-    await collectFlows(agentUuid, timestamp, siteUuid, netify, label, ipToMacMap);
+  } catch (err) {
-
+    console.error(`[Collector] ✗ Error collecting for ${label}:`, err.message);
-    // 5. Threats
+    return { success: false, agent_uuid: agentUuid, error: err.message };
-    await collectThreats(agentUuid, timestamp, siteUuid, netify, label);
+  }
-
+}
-    // 6. Events
+
-    await collectEvents(agentUuid, timestamp, siteUuid, netify, label);
+async function collectAllAgents() {
-
+  const timestamp = new Date();
-    return { success: true, agent_uuid: agentUuid };
+  const timeString = timestamp.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
-  } catch (err) {
+  console.log(`[Collector] === MODE: ALL AGENTS === Started at ${timeString}`);
-    console.error(`[Collector] ✗ Error collecting for ${label}:`, err.message);
+  
-    return { success: false, agent_uuid: agentUuid, error: err.message };
+  const results = [];
-  }
+  let totalAgents = 0;
-}
+  
-
+  if (SITE_UUIDS.length === 0) {
-async function collectAllAgents() {
+    console.warn('[Collector] No NETIFY_SITE_UUIDS configured.');
-  const timestamp = new Date();
+    return { success: false, mode: 'all', message: 'No sites configured', results: [] };
-  const timeString = timestamp.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
+  }
-  console.log(`[Collector] === MODE: ALL AGENTS === Started at ${timeString}`);
+
-  
+  // Track agent UUIDs already assigned to a site to prevent cross-site duplication.
-  const results = [];
+  // The Netify /data/stats/top/agent/download endpoint is org-level and can return
-  let totalAgents = 0;
+  // the same agent for multiple site queries. Each agent must belong to exactly one site.
-  
+  const processedAgentUuids = new Set();
-  if (SITE_UUIDS.length === 0) {
+
-    console.warn('[Collector] No NETIFY_SITE_UUIDS configured.');
+  for (const siteUuid of SITE_UUIDS) {
-    return { success: false, mode: 'all', message: 'No sites configured', results: [] };
+    console.log(`[Collector] Fetching agents for Site: ${siteUuid}`);
-  }
+    const rawAgents = await netify.fetchAgents(siteUuid);
-
+    if (!rawAgents || rawAgents.length === 0) {
-  // Track agent UUIDs already assigned to a site to prevent cross-site duplication.
+      console.warn(`[Collector] No agents found for site ${siteUuid}.`);
-  // The Netify /data/stats/top/agent/download endpoint is org-level and can return
+      continue;
-  // the same agent for multiple site queries. Each agent must belong to exactly one site.
+    }
-  const processedAgentUuids = new Set();
+    
-
+    // Deduplicate: only keep agents not yet seen in a previous site this cycle
-  for (const siteUuid of SITE_UUIDS) {
+    const agents = rawAgents.filter(a => {
-    console.log(`[Collector] Fetching agents for Site: ${siteUuid}`);
+      if (processedAgentUuids.has(a.uuid)) {
-    const rawAgents = await netify.fetchAgents(siteUuid);
+        console.log(`[Collector] Skipping agent ${a.uuid} — already assigned to another site.`);
-    if (!rawAgents || rawAgents.length === 0) {
+        return false;
-      console.warn(`[Collector] No agents found for site ${siteUuid}.`);
+      }
-      continue;
+      return true;
-    }
+    });
-    
+
-    // Deduplicate: only keep agents not yet seen in a previous site this cycle
+    if (agents.length === 0) {
-    const agents = rawAgents.filter(a => {
+      console.warn(`[Collector] No unique agents for site ${siteUuid} (all were already assigned). Skipping.`);
-      if (processedAgentUuids.has(a.uuid)) {
+      continue;
-        console.log(`[Collector] Skipping agent ${a.uuid} — already assigned to another site.`);
+    }
-        return false;
+
-      }
+    // Register these agents as belonging to this site
-      return true;
+    for (const agent of agents) processedAgentUuids.add(agent.uuid);
-    });
+
-
+    // ── Upsert all agents into agent_registry collection ───────────────────
-    if (agents.length === 0) {
+    // This ensures ALL agents appear in the frontend even with no telemetry data.
-      console.warn(`[Collector] No unique agents for site ${siteUuid} (all were already assigned). Skipping.`);
+    await Promise.allSettled(agents.map(a =>
-      continue;
+      AgentRegistry.findOneAndUpdate(
-    }
+        { uuid: a.uuid },
-
+        {
-    // Register these agents as belonging to this site
+          $set: {
-    for (const agent of agents) processedAgentUuids.add(agent.uuid);
+            uuid:         a.uuid,
-
+            serial:       a.serial || a.uuid,
-    totalAgents += agents.length;
+            label:        a.label,
-    console.log(`[Collector] Processing ${agents.length} agents for site ${siteUuid}: ${agents.map(a => a.uuid).join(', ')}`);
+            site_uuid:    siteUuid,
-
+            provisioned:  a.provisioned ?? true,
-    // ── Site-Level Summary (Pilihan A) ─────────────────────────────────────
+            activated:    a.activated ?? false,
-    // Collect bandwidth at site level (no agentUuid filter) so numbers match
+            last_seen_at: a.last_seen_at ?? null,
-    // Netify portal exactly and avoid double-counting across agents.
+            netify_id:    a.id ?? null,
-    try {
+          }
-      console.log(`[Collector] → Fetching site-level summary for site: ${siteUuid}`);
+        },
-      const siteSummary = await netify.fetchBandwidthSummary(1440, null, siteUuid);
+        { upsert: true, new: true }
-      if (siteSummary) {
+      )
-        let download_speed = 0;
+    ));
-        let upload_speed = 0;
+    console.log(`[Collector]   ✓ ${agents.length} agents upserted into registry for site ${siteUuid}`);
-
+
-        try {
+    totalAgents += agents.length;
-          const prev = await Summary.findOne({ agent_uuid: null, site_uuid: siteUuid }).sort({ timestamp: -1 }).lean();
+    console.log(`[Collector] Processing ${agents.length} agents for site ${siteUuid}: ${agents.map(a => a.uuid).join(', ')}`);
-          if (prev && prev.timestamp) {
+
-            const timeDiffSec = (timestamp.getTime() - new Date(prev.timestamp).getTime()) / 1000;
+    // ── Site-Level Summary (Pilihan A) ─────────────────────────────────────
-            if (timeDiffSec > 0) {
+    // Collect bandwidth at site level (no agentUuid filter) so numbers match
-              const bytesDiffDown = Math.max(0, siteSummary.bandwidth_down - (prev.bandwidth_down || 0));
+    // Netify portal exactly and avoid double-counting across agents.
-              const bytesDiffUp   = Math.max(0, siteSummary.bandwidth_up - (prev.bandwidth_up || 0));
+    try {
-              download_speed = bytesDiffDown / timeDiffSec;
+      const siteSummary = await netify.fetchBandwidthSummary(5, null, siteUuid);
-              upload_speed   = bytesDiffUp / timeDiffSec;
+      if (siteSummary) {
-            }
+        let download_speed = 0;
-          }
+        let upload_speed = 0;
-        } catch (err) {
+
-          console.error('[Collector] Error calculating site summary speeds:', err.message);
+        try {
-        }
+          const prev = await Summary.findOne({ agent_uuid: null, site_uuid: siteUuid }).sort({ timestamp: -1 }).lean();
-
+          const timeDiffSec = prev && prev.timestamp ? (timestamp.getTime() - new Date(prev.timestamp).getTime()) / 1000 : 300;
-        const activeFlows = siteSummary.active_flows || 0;
+          const activeTimeDiff = timeDiffSec > 0 ? timeDiffSec : 300;
-        const totalBandwidth = (siteSummary.bandwidth_down || 0) + (siteSummary.bandwidth_up || 0);
+          download_speed = siteSummary.bandwidth_down / activeTimeDiff;
-        const packet_drops = Math.floor(activeFlows * 0.015);
+          upload_speed   = siteSummary.bandwidth_up / activeTimeDiff;
-        const peak_flow_rate = Math.floor(activeFlows * 1.18);
+        } catch (err) {
-        const cpu_usage = Math.min(98, Math.max(1.2, parseFloat((2.5 + (activeFlows * 0.04) + (totalBandwidth / 10000000)).toFixed(2))));
+          console.error('[Collector] Error calculating site summary speeds:', err.message);
-        const memory_usage = Math.min(99, Math.max(10.5, parseFloat((15.4 + (activeFlows * 0.02) + (totalBandwidth / 25000000)).toFixed(2))));
+        }
-        const queue_depth = Math.max(0, Math.floor((activeFlows * 0.15) + (totalBandwidth / 5000000)));
+
-
+        const activeFlows = siteSummary.active_flows || 0;
-        await new Summary({
+        const totalBandwidth = (siteSummary.bandwidth_down || 0) + (siteSummary.bandwidth_up || 0);
-          timestamp,
+        const packet_drops = Math.floor(activeFlows * 0.015);
-          agent_uuid: null,       // null = site-level aggregate (bukan per-agent)
+        const peak_flow_rate = Math.floor(activeFlows * 1.18);
-          site_uuid: siteUuid,
+        const cpu_usage = Math.min(98, Math.max(1.2, parseFloat((2.5 + (activeFlows * 0.04) + (totalBandwidth / 10000000)).toFixed(2))));
-          ...siteSummary,
+        const memory_usage = Math.min(99, Math.max(10.5, parseFloat((15.4 + (activeFlows * 0.02) + (totalBandwidth / 25000000)).toFixed(2))));
-          download_speed,
+        const queue_depth = Math.max(0, Math.floor((activeFlows * 0.15) + (totalBandwidth / 5000000)));
-          upload_speed,
+
-          packet_drops,
+        await new Summary({
-          peak_flow_rate,
+          timestamp,
-          cpu_usage,
+          agent_uuid: null,       // null = site-level aggregate (bukan per-agent)
-          memory_usage,
+          site_uuid: siteUuid,
-          queue_depth
+          ...siteSummary,
-        }).save();
+          download_speed,
-        console.log(`[Collector]   ✓ Site-level summary saved for site: ${siteUuid} | Down: ${(siteSummary.bandwidth_down / 1e9).toFixed(2)} GB | Up: ${(siteSummary.bandwidth_up / 1e9).toFixed(2)} GB | Flows: ${siteSummary.active_flows?.toLocaleString()}`);
+          upload_speed,
-      }
+          packet_drops,
-    } catch (err) {
+          peak_flow_rate,
-      console.error(`[Collector] ✗ Failed to save site-level summary for ${siteUuid}:`, err.message);
+          cpu_usage,
-    }
+          memory_usage,
-
+          queue_depth
-    for (const agent of agents) {
+        }).save();
-      const result = await collectForAgent(agent.uuid, timestamp, siteUuid);
+        console.log(`[Collector]   ✓ Site-level summary saved for site: ${siteUuid} | Down: ${(siteSummary.bandwidth_down / 1e9).toFixed(2)} GB | Up: ${(siteSummary.bandwidth_up / 1e9).toFixed(2)} GB | Flows: ${siteSummary.active_flows?.toLocaleString()}`);
-      results.push({ ...result, agent_label: agent.label, site_uuid: siteUuid });
+      }
-    }
+    } catch (err) {
-  }
+      console.error(`[Collector] ✗ Failed to save site-level summary for ${siteUuid}:`, err.message);
-
+    }
-  const successful = results.filter(r => r.success).length;
+
-  console.log(`[Collector] === ALL AGENTS DONE === ${successful}/${totalAgents} successful across ${SITE_UUIDS.length} sites`);
+    for (const agent of agents) {
-  
+      const result = await collectForAgent(agent.uuid, timestamp, siteUuid);
-  await pruneOldData().catch(err => console.error('[Collector] [Retention] error:', err.message));
+      results.push({ ...result, agent_label: agent.label, site_uuid: siteUuid });
-  
+    }
-  return { success: true, mode: 'all', agents_count: totalAgents, successful };
+  }
-}
+
-
+  const successful = results.filter(r => r.success).length;
-async function collectSpecificAgent(agentUuid, siteUuid = SITE_UUIDS[0]) {
+  console.log(`[Collector] === ALL AGENTS DONE === ${successful}/${totalAgents} successful across ${SITE_UUIDS.length} sites`);
-  const timestamp = new Date();
+  
-  const timeString = timestamp.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
+  await pruneOldData().catch(err => console.error('[Collector] [Retention] error:', err.message));
-  console.log(`[Collector] === MODE: SPECIFIC AGENT ${agentUuid} === Started at ${timeString}`);
+  
-  const result = await collectForAgent(agentUuid, timestamp, siteUuid);
+  return { success: true, mode: 'all', agents_count: totalAgents, successful };
-  
+}
-  await pruneOldData().catch(err => console.error('[Collector] [Retention] error:', err.message));
+
-  
+async function collectSpecificAgent(agentUuid, siteUuid = SITE_UUIDS[0]) {
-  return { success: result.success, mode: 'specific', agent_uuid: agentUuid };
+  const result = await collectSpecificAgents([agentUuid], siteUuid, 0);
-}
+  return { success: result.successful > 0, mode: 'specific', agent_uuid: agentUuid };
-
+}
-async function collectSpecificAgents(agentUuids, siteUuid = SITE_UUIDS[0], delayMs = 5000) {
+
-  const timestamp = new Date();
+async function collectSpecificAgents(agentUuids, siteUuid = SITE_UUIDS[0], delayMs = 5000) {
-  const timeString = timestamp.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
+  const timestamp = new Date();
-  console.log(`[Collector] === MODE: SPECIFIC AGENTS [${agentUuids.join(', ')}] === Started at ${timeString}`);
+  const timeString = timestamp.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
-  const results = [];
+  console.log(`[Collector] === MODE: SPECIFIC AGENTS [${agentUuids.join(', ')}] === Started at ${timeString}`);
-  for (let i = 0; i < agentUuids.length; i++) {
+  const results = [];
-    if (i > 0) {
+  for (let i = 0; i < agentUuids.length; i++) {
-      console.log(`[Collector] Waiting ${delayMs}ms before next agent...`);
+    if (i > 0) {
-      await new Promise(resolve => setTimeout(resolve, delayMs));
+      console.log(`[Collector] Waiting ${delayMs}ms before next agent...`);
-    }
+      await new Promise(resolve => setTimeout(resolve, delayMs));
-    const result = await collectForAgent(agentUuids[i], timestamp, siteUuid);
+    }
-    results.push(result);
+    const result = await collectForAgent(agentUuids[i], timestamp, siteUuid);
-  }
+    results.push(result);
-  const successful = results.filter(r => r.success).length;
+  }
-  console.log(`[Collector] === SPECIFIC AGENTS DONE === ${successful}/${agentUuids.length} successful`);
+  const successful = results.filter(r => r.success).length;
-
+  console.log(`[Collector] === SPECIFIC AGENTS DONE === ${successful}/${agentUuids.length} successful`);
-  await pruneOldData().catch(err => console.error('[Collector] [Retention] error:', err.message));
+
-
+  await pruneOldData().catch(err => console.error('[Collector] [Retention] error:', err.message));
-  return { success: true, mode: 'specific_agents', agents_count: agentUuids.length, successful, results };
+
-}
+  return { success: true, mode: 'specific_agents', agents_count: agentUuids.length, successful, results };
-
+}
-module.exports = {
+
-  collectAllAgents,
+module.exports = {
-  collectSpecificAgent,
+  collectAllAgents,
-  collectSpecificAgents
+  collectSpecificAgent,
-};
+  collectSpecificAgents
+};

proxy/collectorHelper.js

@@ -1,167 +1,167 @@
-// proxy/collectorHelper.js
+// proxy/collectorHelper.js
-// ─────────────────────────────────────────────────────────────────────────────
+// ─────────────────────────────────────────────────────────────────────────────
-// Supplementary Telemetry collection steps for BackOne Proxy Server.
+// Supplementary Telemetry collection steps for BackOne Proxy Server.
-// Split from collector.js to satisfy the 256-line file size limit.
+// Split from collector.js to satisfy the 256-line file size limit.
-// ─────────────────────────────────────────────────────────────────────────────
+// ─────────────────────────────────────────────────────────────────────────────
-
+
-const {
+const {
-  AppCategoryStat,
+  AppCategoryStat,
-  TlsVersionStat,
+  TlsVersionStat,
-  TlsCipherStat,
+  TlsCipherStat,
-  TlsSecurityStat,
+  TlsSecurityStat,
-  CountryStat,
+  CountryStat,
-  ProtocolStat,
+  ProtocolStat,
-  SslSubjectAltNameStat,
+  SslSubjectAltNameStat,
-  SniHostnameStat,
+  SniHostnameStat,
-  SslServerCnStat,
+  SslServerCnStat,
-  QuicHostnameStat,
+  QuicHostnameStat,
-} = require('./models/Schemas');
+} = require('./models/Schemas');
-
+
-async function collectSecondaryTelemetry(agentUuid, timestamp, SITE_UUID, netify, label) {
+async function collectSecondaryTelemetry(agentUuid, timestamp, SITE_UUID, netify, label) {
-  try {
+  try {
-    // 2b. App Categories
+    // 2b. App Categories
-    const categories = await netify.fetchTopAppCategories(1440, 50, agentUuid, SITE_UUID);
+    const categories = await netify.fetchTopAppCategories(5, 50, agentUuid, SITE_UUID);
-    if (categories && categories.length > 0) {
+    if (categories && categories.length > 0) {
-      const catDocs = categories.map(c => ({
+      const catDocs = categories.map(c => ({
-        timestamp,
+        timestamp,
-        agent_uuid:     agentUuid,
+        agent_uuid:     agentUuid,
-        site_uuid:      SITE_UUID,
+        site_uuid:      SITE_UUID,
-        category_label: c.category_label,
+        category_label: c.category_label,
-        download:       c.download || 0,
+        download:       c.download || 0,
-        upload:         c.upload || 0,
+        upload:         c.upload || 0,
-        flows:          c.flows || 0,
+        flows:          c.flows || 0,
-      }));
+      }));
-      await AppCategoryStat.insertMany(catDocs);
+      await AppCategoryStat.insertMany(catDocs);
-      console.log(`[Collector]   ✓ ${catDocs.length} categories saved for ${label}`);
+      console.log(`[Collector]   ✓ ${catDocs.length} categories saved for ${label}`);
-    }
+    }
-
+
-    // 2c. TLS Versions
+    // 2c. TLS Versions
-    const tlsVersions = await netify.fetchTlsVersions(1440, 50, agentUuid, SITE_UUID);
+    const tlsVersions = await netify.fetchTlsVersions(5, 50, agentUuid, SITE_UUID);
-    if (tlsVersions && tlsVersions.length > 0) {
+    if (tlsVersions && tlsVersions.length > 0) {
-      const tvDocs = tlsVersions.map(v => ({
+      const tvDocs = tlsVersions.map(v => ({
-        timestamp,
+        timestamp,
-        agent_uuid:   agentUuid,
+        agent_uuid:   agentUuid,
-        site_uuid:    SITE_UUID,
+        site_uuid:    SITE_UUID,
-        tls_version:  v.tls_version,
+        tls_version:  v.tls_version,
-        download:     v.download || 0,
+        download:     v.download || 0,
-        upload:       v.upload || 0,
+        upload:       v.upload || 0,
-        flows:        v.flows || 0,
+        flows:        v.flows || 0,
-      }));
+      }));
-      await TlsVersionStat.insertMany(tvDocs);
+      await TlsVersionStat.insertMany(tvDocs);
-      console.log(`[Collector]   ✓ ${tvDocs.length} TLS versions saved for ${label}`);
+      console.log(`[Collector]   ✓ ${tvDocs.length} TLS versions saved for ${label}`);
-    }
+    }
-
+
-    // 2d. TLS Ciphers
+    // 2d. TLS Ciphers
-    const tlsCiphers = await netify.fetchTlsCiphers(1440, 50, agentUuid, SITE_UUID);
+    const tlsCiphers = await netify.fetchTlsCiphers(5, 50, agentUuid, SITE_UUID);
-    if (tlsCiphers && tlsCiphers.length > 0) {
+    if (tlsCiphers && tlsCiphers.length > 0) {
-      const tcDocs = tlsCiphers.map(c => ({
+      const tcDocs = tlsCiphers.map(c => ({
-        timestamp,
+        timestamp,
-        agent_uuid:   agentUuid,
+        agent_uuid:   agentUuid,
-        site_uuid:    SITE_UUID,
+        site_uuid:    SITE_UUID,
-        tls_cipher:   c.tls_cipher,
+        tls_cipher:   c.tls_cipher,
-        download:     c.download || 0,
+        download:     c.download || 0,
-        upload:       c.upload || 0,
+        upload:       c.upload || 0,
-        flows:        c.flows || 0,
+        flows:        c.flows || 0,
-      }));
+      }));
-      await TlsCipherStat.insertMany(tcDocs);
+      await TlsCipherStat.insertMany(tcDocs);
-      console.log(`[Collector]   ✓ ${tcDocs.length} TLS ciphers saved for ${label}`);
+      console.log(`[Collector]   ✓ ${tcDocs.length} TLS ciphers saved for ${label}`);
-    }
+    }
-
+
-    // 2e. TLS Security
+    // 2e. TLS Security
-    const tlsSecurity = await netify.fetchTlsSecurity(1440, 50, agentUuid, SITE_UUID);
+    const tlsSecurity = await netify.fetchTlsSecurity(5, 50, agentUuid, SITE_UUID);
-    if (tlsSecurity && tlsSecurity.length > 0) {
+    if (tlsSecurity && tlsSecurity.length > 0) {
-      const tsDocs = tlsSecurity.map(s => ({
+      const tsDocs = tlsSecurity.map(s => ({
-        timestamp,
+        timestamp,
-        agent_uuid:   agentUuid,
+        agent_uuid:   agentUuid,
-        site_uuid:    SITE_UUID,
+        site_uuid:    SITE_UUID,
-        tls_security: s.tls_security,
+        tls_security: s.tls_security,
-        download:     s.download || 0,
+        download:     s.download || 0,
-        upload:       s.upload || 0,
+        upload:       s.upload || 0,
-        flows:        s.flows || 0,
+        flows:        s.flows || 0,
-      }));
+      }));
-      await TlsSecurityStat.insertMany(tsDocs);
+      await TlsSecurityStat.insertMany(tsDocs);
-      console.log(`[Collector]   ✓ ${tsDocs.length} TLS security stats saved for ${label}`);
+      console.log(`[Collector]   ✓ ${tsDocs.length} TLS security stats saved for ${label}`);
-    }
+    }
-
+
-    // 2f. Top Countries
+    // 2f. Top Countries
-    const countries = await netify.fetchTopCountries(1440, 100, agentUuid, SITE_UUID);
+    const countries = await netify.fetchTopCountries(5, 100, agentUuid, SITE_UUID);
-    if (countries && countries.length > 0) {
+    if (countries && countries.length > 0) {
-      const coDocs = countries.map(c => ({
+      const coDocs = countries.map(c => ({
-        timestamp,
+        timestamp,
-        agent_uuid:   agentUuid,
+        agent_uuid:   agentUuid,
-        site_uuid:    SITE_UUID,
+        site_uuid:    SITE_UUID,
-        country_code: c.country_code,
+        country_code: c.country_code,
-        country_name: c.country_name || '',
+        country_name: c.country_name || '',
-        download:     c.download || 0,
+        download:     c.download || 0,
-        upload:       c.upload   || 0,
+        upload:       c.upload   || 0,
-        flows:        c.flows    || 0,
+        flows:        c.flows    || 0,
-      }));
+      }));
-      await CountryStat.insertMany(coDocs);
+      await CountryStat.insertMany(coDocs);
-      console.log(`[Collector]   ✓ ${coDocs.length} countries saved for ${label}`);
+      console.log(`[Collector]   ✓ ${coDocs.length} countries saved for ${label}`);
-    }
+    }
-
+
-    // 2g. Top Protocols
+    // 2g. Top Protocols
-    const protocols = await netify.fetchTopProtocols(1440, 50, agentUuid, SITE_UUID);
+    const protocols = await netify.fetchTopProtocols(5, 50, agentUuid, SITE_UUID);
-    if (protocols && protocols.length > 0) {
+    if (protocols && protocols.length > 0) {
-      const protoDocs = protocols.map(p => ({
+      const protoDocs = protocols.map(p => ({
-        timestamp,
+        timestamp,
-        agent_uuid:     agentUuid,
+        agent_uuid:     agentUuid,
-        site_uuid:      SITE_UUID,
+        site_uuid:      SITE_UUID,
-        protocol_label: p.protocol_label,
+        protocol_label: p.protocol_label,
-        download:       p.download || 0,
+        download:       p.download || 0,
-        upload:         p.upload || 0,
+        upload:         p.upload || 0,
-        flows:          p.flows || 0,
+        flows:          p.flows || 0,
-      }));
+      }));
-      await ProtocolStat.insertMany(protoDocs);
+      await ProtocolStat.insertMany(protoDocs);
-      console.log(`[Collector]   ✓ ${protoDocs.length} protocols saved for ${label}`);
+      console.log(`[Collector]   ✓ ${protoDocs.length} protocols saved for ${label}`);
-    }
+    }
-
+
-    // 2h. SNI Hostnames
+    // 2h. SNI Hostnames
-    const snis = await netify.fetchSniHostnames(1440, 10000, agentUuid, SITE_UUID);
+    const snis = await netify.fetchSniHostnames(5, 10000, agentUuid, SITE_UUID);
-    if (snis && snis.length > 0) {
+    if (snis && snis.length > 0) {
-      const sniDocs = snis.map(s => ({
+      const sniDocs = snis.map(s => ({
-        timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
+        timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
-        sni_hostname: (s.sni_hostname && String(s.sni_hostname).trim() !== '') ? s.sni_hostname : 'Unknown',
+        sni_hostname: (s.sni_hostname && String(s.sni_hostname).trim() !== '') ? s.sni_hostname : 'Unknown',
-        download: s.download || 0, upload: s.upload || 0, flows: s.flows || 0,
+        download: s.download || 0, upload: s.upload || 0, flows: s.flows || 0,
-      }));
+      }));
-      await SniHostnameStat.insertMany(sniDocs);
+      await SniHostnameStat.insertMany(sniDocs);
-      console.log(`[Collector]   ✓ ${sniDocs.length} SNI hostnames saved for ${label}`);
+      console.log(`[Collector]   ✓ ${sniDocs.length} SNI hostnames saved for ${label}`);
-    }
+    }
-
+
-    /* -- COMMENTED OUT DUE TO NETIFY API HTTP 422 (UNSUPPORTED TIER) --
+    /* -- COMMENTED OUT DUE TO NETIFY API HTTP 422 (UNSUPPORTED TIER) --
-    // 2i. SSL Server Common Names
+    // 2i. SSL Server Common Names
-    const cns = await netify.fetchSslServerCn(1440, 50, agentUuid);
+    const cns = await netify.fetchSslServerCn(1440, 50, agentUuid);
-    if (cns && cns.length > 0) {
+    if (cns && cns.length > 0) {
-      const cnDocs = cns.map(c => ({
+      const cnDocs = cns.map(c => ({
-        timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
+        timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
-        ssl_server_cn: c.ssl_server_cn, download: c.download || 0, upload: c.upload || 0, flows: c.flows || 0,
+        ssl_server_cn: c.ssl_server_cn, download: c.download || 0, upload: c.upload || 0, flows: c.flows || 0,
-      }));
+      }));
-      await SslServerCnStat.insertMany(cnDocs);
+      await SslServerCnStat.insertMany(cnDocs);
-      console.log(`[Collector]   ✓ ${cnDocs.length} SSL Server CNs saved for ${label}`);
+      console.log(`[Collector]   ✓ ${cnDocs.length} SSL Server CNs saved for ${label}`);
-    }
+    }
-
+
-    // 2j. QUIC Hostnames
+    // 2j. QUIC Hostnames
-    const quics = await netify.fetchQuicHostnames(1440, 50, agentUuid);
+    const quics = await netify.fetchQuicHostnames(1440, 50, agentUuid);
-    if (quics && quics.length > 0) {
+    if (quics && quics.length > 0) {
-      const quicDocs = quics.map(q => ({
+      const quicDocs = quics.map(q => ({
-        timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
+        timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
-        quic_hostname: q.quic_hostname, download: q.download || 0, upload: q.upload || 0, flows: q.flows || 0,
+        quic_hostname: q.quic_hostname, download: q.download || 0, upload: q.upload || 0, flows: q.flows || 0,
-      }));
+      }));
-      await QuicHostnameStat.insertMany(quicDocs);
+      await QuicHostnameStat.insertMany(quicDocs);
-      console.log(`[Collector]   ✓ ${quicDocs.length} QUIC hostnames saved for ${label}`);
+      console.log(`[Collector]   ✓ ${quicDocs.length} QUIC hostnames saved for ${label}`);
-    }
+    }
-    */
+    */
-
+
-    // NOTE: The following API fields are not supported on this subscription (HTTP 422):
+    // NOTE: The following API fields are not supported on this subscription (HTTP 422):
-    //   dhcp_class, http_useragent, bittorrent_info_hash, ssl_subject_alt_name
+    //   dhcp_class, http_useragent, bittorrent_info_hash, ssl_subject_alt_name
-    // These sections are intentionally skipped to avoid wasted API calls.
+    // These sections are intentionally skipped to avoid wasted API calls.
-    // Re-enable when API access is upgraded to a tier that supports these fields.
+    // Re-enable when API access is upgraded to a tier that supports these fields.
-
+
-  } catch (err) {
+  } catch (err) {
-    console.error(`[CollectorHelper] Error saving secondary telemetry:`, err.message);
+    console.error(`[CollectorHelper] Error saving secondary telemetry:`, err.message);
-  }
+  }
-}
+}
-
+
-module.exports = {
+module.exports = {
-  collectSecondaryTelemetry,
+  collectSecondaryTelemetry,
-};
+};

proxy/collectorHelperDpi2.js

@@ -14,7 +14,7 @@ const {
 } = require('./deviceResolver');
 
 async function collectDevicesAndApps(agentUuid, timestamp, SITE_UUID, netify, label) {
-  const devices = await netify.fetchDiscoveredDevices(1440, 500, agentUuid, SITE_UUID);
+  const devices = await netify.fetchDiscoveredDevices(5, 500, agentUuid, SITE_UUID);
   const ipToMacMap = {};
   
   if (devices && devices.length > 0) {
@@ -56,7 +56,7 @@ async function collectDevicesAndApps(agentUuid, timestamp, SITE_UUID, netify, la
       let deviceAppCount = 0;
       for (let i = 0; i < topDevices.length; i += 5) {
         const batch = topDevices.slice(i, i + 5);
-        const results = await Promise.allSettled(batch.map(d => netify.fetchDeviceApps(d.ip_address, 1440, 50, agentUuid, SITE_UUID)));
+        const results = await Promise.allSettled(batch.map(d => netify.fetchDeviceApps(d.ip_address, 5, 50, agentUuid, SITE_UUID)));
         const appDocs = [];
         results.forEach((res, idx) => {
           if (res.status === 'fulfilled' && Array.isArray(res.value)) {
@@ -81,7 +81,7 @@ async function collectDevicesAndApps(agentUuid, timestamp, SITE_UUID, netify, la
 }
 
 async function collectFlows(agentUuid, timestamp, SITE_UUID, netify, label, ipToMacMap) {
-  // Netify API has a hard limit of 1,000,000 for settings_limit.
+  // Netify API has a hard limit of 1,000,000 for settings_limit. Use 1000000 as default per rule.
   const flowLimit = parseInt(process.env.PROXY_FLOW_LIMIT || '1000000');
   const flows = await netify.fetchFlows(flowLimit, agentUuid, SITE_UUID);
   if (flows && flows.length > 0) {
@@ -115,16 +115,34 @@ async function collectFlows(agentUuid, timestamp, SITE_UUID, netify, label, ipTo
           const blacklistedCategories = new Set(blacklistRules.filter(r => r.type === 'category').map(r => r.value.toLowerCase()));
           const blacklistedDomains = new Set(blacklistRules.filter(r => r.type === 'domain').map(r => r.value.toLowerCase()));
 
+          const flowIdsInBatch = flowDocs.map(f => f.flow_id).filter(Boolean);
+          const existingFlowThreats = new Set(
+            await Threat.find({ flow_id: { $in: flowIdsInBatch } }).distinct('flow_id')
+          );
+
           const threatDocs = [];
+          const eventDocs = [];
+
           for (const f of flowDocs) {
             let isViolation = false;
             let categoryLabel = "";
 
             // Check if domain is blacklisted
-            if (f.domain && blacklistedDomains.has(f.domain.toLowerCase())) {
+            for (const r of blacklistRules) {
-              isViolation = true;
+              if (r.type === 'domain') {
-            } else if (f.app_label && blacklistedDomains.has(f.app_label.toLowerCase())) {
+                const val = r.value.toLowerCase();
-              isViolation = true;
+                // Direct domain match
+                if (f.domain && f.domain.toLowerCase().includes(val)) {
+                  isViolation = true;
+                  break;
+                }
+                // Main domain part match against app label (e.g. "google" from "google.com")
+                const mainDomainPart = val.split('.')[0];
+                if (mainDomainPart && f.app_label && f.app_label.toLowerCase().includes(mainDomainPart)) {
+                  isViolation = true;
+                  break;
+                }
+              }
             }
 
             // Look up app details to check category
@@ -138,7 +156,7 @@ async function collectFlows(agentUuid, timestamp, SITE_UUID, netify, label, ipTo
               }
             }
 
-            if (isViolation) {
+            if (isViolation && !existingFlowThreats.has(f.flow_id)) {
               threatDocs.push({
                 timestamp,
                 agent_uuid: f.agent_uuid,
@@ -150,7 +168,21 @@ async function collectFlows(agentUuid, timestamp, SITE_UUID, netify, label, ipTo
                 dst_port: f.dst_port,
                 protocol: f.protocol,
                 description: `Access to blacklisted app/domain: ${f.app_label} (${f.domain || 'N/A'})${categoryLabel ? ' - Category: ' + categoryLabel : ''}`,
-                event_at: new Date().toISOString()
+                event_at: new Date().toISOString(),
+                flow_id: f.flow_id
+              });
+
+              eventDocs.push({
+                timestamp,
+                agent_uuid: f.agent_uuid,
+                site_uuid: f.site_uuid,
+                event_type: "blacklist_violation",
+                severity: "Warning",
+                description: `Access to blacklisted app/domain: ${f.app_label} (${f.domain || 'N/A'})${categoryLabel ? ' - Category: ' + categoryLabel : ''}`,
+                ip_address: f.src_ip,
+                mac_address: f.src_mac,
+                event_at: new Date(),
+                flow_id: f.flow_id
               });
             }
           }
@@ -159,6 +191,10 @@ async function collectFlows(agentUuid, timestamp, SITE_UUID, netify, label, ipTo
             await Threat.insertMany(threatDocs);
             console.log(`[Collector]   ✓ ${threatDocs.length} blacklist policy violation threats recorded for ${label}`);
           }
+          if (eventDocs.length > 0) {
+            await Event.insertMany(eventDocs);
+            console.log(`[Collector]   ✓ ${eventDocs.length} blacklist policy violation events recorded for ${label}`);
+          }
         }
       } catch (err) {
         console.error('[Collector] Blacklist detection failed:', err.message);
@@ -169,58 +205,7 @@ async function collectFlows(agentUuid, timestamp, SITE_UUID, netify, label, ipTo
   }
 }
 
-async function collectThreats(agentUuid, timestamp, SITE_UUID, netify, label) {
-  const threats = await netify.fetchCyberThreats(agentUuid, SITE_UUID);
-  if (threats && threats.length > 0) {
-    const threatDocs = threats.map(t => ({
-      timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
-      threat_type: t.threat_type || 'Unknown Threat', severity: t.severity || 'Medium',
-      src_ip: t.src_ip, dst_ip: t.dst_ip, dst_port: t.dst_port, protocol: t.protocol,
-      description: t.description, event_at: t.event_at || new Date().toISOString(),
-    }));
-    await Threat.insertMany(threatDocs);
-    console.log(`[Collector]   ✓ ${threatDocs.length} threats saved for ${label}`);
-  }
-}
-
-async function collectEvents(agentUuid, timestamp, SITE_UUID, netify, label) {
-  const events = await netify.fetchEvents(100, agentUuid, SITE_UUID);
-  if (events && events.length > 0) {
-    const eventIds = events.map(e => e.event_id).filter(id => id !== null);
-    const existing = await Event.find({ site_uuid: SITE_UUID, event_id: { $in: eventIds } }).distinct('event_id');
-    const existingSet = new Set(existing);
-
-    const macToAgentMap = {};
-    const eventMacs = [...new Set(events.map(e => e.mac_address).filter(Boolean))];
-    if (eventMacs.length > 0) {
-      const storedDevices = await DeviceStat.find(
-        { site_uuid: SITE_UUID, mac_address: { $in: eventMacs } },
-        { mac_address: 1, agent_uuid: 1 }
-      ).lean();
-      for (const d of storedDevices) {
-        if (d.mac_address && d.agent_uuid) macToAgentMap[d.mac_address] = d.agent_uuid;
-      }
-    }
-
-    const eventDocs = events.filter(e => e.event_id === null || !existingSet.has(e.event_id)).map(e => {
-      const resolvedAgentUuid = (e.mac_address && macToAgentMap[e.mac_address]) || agentUuid;
-      return {
-        timestamp, agent_uuid: resolvedAgentUuid, site_uuid: SITE_UUID,
-        event_id: e.event_id, event_type: e.event_type, severity: e.severity,
-        description: e.description, category_label: e.category_label,
-        ip_address: e.ip_address, mac_address: e.mac_address, event_at: e.event_at,
-      };
-    });
-    if (eventDocs.length > 0) {
-      await Event.insertMany(eventDocs);
-      console.log(`[Collector]   ✓ ${eventDocs.length} new events saved for ${label}`);
-    }
-  }
-}
-
 module.exports = {
   collectDevicesAndApps,
-  collectFlows,
+  collectFlows
-  collectThreats,
-  collectEvents
 };

proxy/collectorHelperDpi3.js

@@ -0,0 +1,61 @@
+// proxy/collectorHelperDpi3.js
+// ─────────────────────────────────────────────────────────────────────────────
+// Supplementary Telemetry collection steps for threats and events.
+// Split from collectorHelperDpi2.js to satisfy the 256-line file size limit.
+// ─────────────────────────────────────────────────────────────────────────────
+
+const { DeviceStat, Threat, Event } = require('./models/Schemas');
+
+async function collectThreats(agentUuid, timestamp, SITE_UUID, netify, label) {
+  const threats = await netify.fetchCyberThreats(agentUuid, SITE_UUID);
+  if (threats && threats.length > 0) {
+    const threatDocs = threats.map(t => ({
+      timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
+      threat_type: t.threat_type || 'Unknown Threat', severity: t.severity || 'Medium',
+      src_ip: t.src_ip, dst_ip: t.dst_ip, dst_port: t.dst_port, protocol: t.protocol,
+      description: t.description, event_at: t.event_at || new Date().toISOString(),
+    }));
+    await Threat.insertMany(threatDocs);
+    console.log(`[Collector]   ✓ ${threatDocs.length} threats saved for ${label}`);
+  }
+}
+
+async function collectEvents(agentUuid, timestamp, SITE_UUID, netify, label) {
+  const events = await netify.fetchEvents(100, agentUuid, SITE_UUID);
+  if (events && events.length > 0) {
+    const eventIds = events.map(e => e.event_id).filter(id => id !== null);
+    const existing = await Event.find({ site_uuid: SITE_UUID, event_id: { $in: eventIds } }).distinct('event_id');
+    const existingSet = new Set(existing);
+
+    const macToAgentMap = {};
+    const eventMacs = [...new Set(events.map(e => e.mac_address).filter(Boolean))];
+    if (eventMacs.length > 0) {
+      const storedDevices = await DeviceStat.find(
+        { site_uuid: SITE_UUID, mac_address: { $in: eventMacs } },
+        { mac_address: 1, agent_uuid: 1 }
+      ).lean();
+      for (const d of storedDevices) {
+        if (d.mac_address && d.agent_uuid) macToAgentMap[d.mac_address] = d.agent_uuid;
+      }
+    }
+
+    const eventDocs = events.filter(e => e.event_id === null || !existingSet.has(e.event_id)).map(e => {
+      const resolvedAgentUuid = (e.mac_address && macToAgentMap[e.mac_address]) || agentUuid;
+      return {
+        timestamp, agent_uuid: resolvedAgentUuid, site_uuid: SITE_UUID,
+        event_id: e.event_id, event_type: e.event_type, severity: e.severity,
+        description: e.description, category_label: e.category_label,
+        ip_address: e.ip_address, mac_address: e.mac_address, event_at: e.event_at,
+      };
+    });
+    if (eventDocs.length > 0) {
+      await Event.insertMany(eventDocs);
+      console.log(`[Collector]   ✓ ${eventDocs.length} new events saved for ${label}`);
+    }
+  }
+}
+
+module.exports = {
+  collectThreats,
+  collectEvents
+};

proxy/dataRetention.js

@@ -11,9 +11,9 @@ const Schemas = require('./models/Schemas');
  * Prune all time-series documents older than 7 days.
  */
 async function pruneOldData() {
-  const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
+  const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
-  const timeString = sevenDaysAgo.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
+  const timeString = thirtyDaysAgo.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
-  console.log(`[Collector] [Retention] Checking for telemetry data older than 7 days (before ${timeString})...`);
+  console.log(`[Collector] [Retention] Checking for telemetry data older than 30 days (before ${timeString})...`);
 
   // Prune from all collections in Schemas except CustomDeviceLabel
   const collections = Object.keys(Schemas).filter(name => name !== 'CustomDeviceLabel');
@@ -22,7 +22,7 @@ async function pruneOldData() {
     try {
       const Model = Schemas[name];
       if (typeof Model.deleteMany === 'function') {
-        const res = await Model.deleteMany({ timestamp: { $lt: sevenDaysAgo } });
+        const res = await Model.deleteMany({ timestamp: { $lt: thirtyDaysAgo } });
         if (res.deletedCount > 0) {
           console.log(`[Collector] [Retention]   ✓ Cleaned up ${res.deletedCount} old records from ${name}`);
         }

proxy/diagnostic.js

@@ -0,0 +1,44 @@
+// diagnostic.js - Run: node proxy/diagnostic.js
+const path = require('path');
+require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
+const mongoose = require('mongoose');
+
+const MONGODB_URI = process.env.MONGODB_URI || 'mongodb://localhost:27017/backone';
+
+async function run() {
+  await mongoose.connect(MONGODB_URI);
+  const db = mongoose.connection.db;
+
+  // 1. Total count
+  const total = await db.collection('devicestats').countDocuments();
+  console.log('=== DeviceStat Total:', total);
+
+  // 2. Count for 10.6.10.44
+  const specific = await db.collection('devicestats').countDocuments({ ip_address: '10.6.10.44' });
+  console.log('=== Count for 10.6.10.44:', specific);
+
+  // 3. Sample doc for 10.6.10.44
+  const sample = await db.collection('devicestats').findOne({ ip_address: '10.6.10.44' });
+  console.log('=== Sample doc for 10.6.10.44:', JSON.stringify(sample, null, 2));
+
+  // 4. Duplicate groups (top 10)
+  const dups = await db.collection('devicestats').aggregate([
+    { $group: { _id: { agent_uuid: '$agent_uuid', ip_address: '$ip_address' }, count: { $sum: 1 } } },
+    { $match: { count: { $gt: 1 } } },
+    { $sort: { count: -1 } },
+    { $limit: 10 }
+  ]).toArray();
+  console.log('=== Top duplicate groups:', JSON.stringify(dups, null, 2));
+
+  // 5. Check what collection name is actually used
+  const collections = await db.listCollections().toArray();
+  console.log('=== Collections:', collections.map(c => c.name));
+
+  // 6. Check indexes on devicestats
+  const indexes = await db.collection('devicestats').indexes();
+  console.log('=== Indexes on devicestats:', JSON.stringify(indexes, null, 2));
+
+  await mongoose.disconnect();
+}
+
+run().catch(err => { console.error('ERROR:', err.message); process.exit(1); });

proxy/fix_devicestat_index.js

@@ -0,0 +1,79 @@
+// fix_devicestat_index.js
+// Creates a unique compound index on (agent_uuid, ip_address) in DeviceStat
+// and deduplicates any remaining duplicates before creating the index.
+// Run: node proxy/fix_devicestat_index.js
+const path = require('path');
+require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
+const mongoose = require('mongoose');
+
+const MONGODB_URI = process.env.MONGODB_URI || 'mongodb://localhost:27017/backone';
+
+async function run() {
+  console.log('[Fix] Connecting to MongoDB...');
+  await mongoose.connect(MONGODB_URI);
+  const db = mongoose.connection.db;
+  const col = db.collection('devicestats');
+
+  // Step 1: Find all duplicates grouped by (agent_uuid, ip_address)
+  console.log('[Fix] Scanning for duplicates...');
+  const groups = await col.aggregate([
+    {
+      $group: {
+        _id: { agent_uuid: '$agent_uuid', ip_address: '$ip_address' },
+        ids: { $push: '$_id' },
+        timestamps: { $push: '$timestamp' },
+        count: { $sum: 1 },
+      }
+    },
+    { $match: { count: { $gt: 1 } } },
+  ]).toArray();
+
+  console.log(`[Fix] Found ${groups.length} duplicate groups.`);
+  let deleted = 0;
+
+  for (const group of groups) {
+    // Sort by timestamp descending — keep the newest
+    const paired = group.ids.map((id, i) => ({ id, ts: new Date(group.timestamps[i] || 0) }));
+    paired.sort((a, b) => b.ts - a.ts);
+    const toDelete = paired.slice(1).map(p => p.id);
+    const result = await col.deleteMany({ _id: { $in: toDelete } });
+    deleted += result.deletedCount;
+  }
+
+  console.log(`[Fix] Deleted ${deleted} duplicate documents.`);
+  const remaining = await col.countDocuments();
+  console.log(`[Fix] Remaining DeviceStat documents: ${remaining}`);
+
+  // Step 2: Drop old non-unique compound index if it exists
+  try {
+    await col.dropIndex('agent_uuid_1_timestamp_-1_ip_address_1');
+    console.log('[Fix] Dropped old compound index.');
+  } catch (e) {
+    console.log('[Fix] Old index not found or already dropped:', e.message);
+  }
+
+  // Step 3: Create UNIQUE compound index on (agent_uuid, ip_address)
+  try {
+    await col.createIndex(
+      { agent_uuid: 1, ip_address: 1 },
+      { unique: true, name: 'agent_uuid_1_ip_address_1_unique', background: true }
+    );
+    console.log('[Fix] Created unique index on (agent_uuid, ip_address).');
+  } catch (e) {
+    console.error('[Fix] Failed to create unique index:', e.message);
+  }
+
+  // Step 4: Verify indexes
+  const indexes = await col.indexes();
+  console.log('[Fix] Current indexes:');
+  indexes.forEach(idx => console.log(`  - ${idx.name}: ${JSON.stringify(idx.key)} ${idx.unique ? '[UNIQUE]' : ''}`));
+
+  // Step 5: Verify 10.6.10.44
+  const cnt = await col.countDocuments({ ip_address: '10.6.10.44' });
+  console.log(`\n[Fix] Count for 10.6.10.44: ${cnt} (should be 1)`);
+
+  await mongoose.disconnect();
+  console.log('[Fix] Done.');
+}
+
+run().catch(err => { console.error('[Fix] Fatal:', err.message); process.exit(1); });

proxy/index.js

@@ -1,10 +1,16 @@
 // proxy/index.js
 // ─────────────────────────────────────────────────────────────────────────────
+// Polyfill global crypto for Node 18 compatibility (required by mongodb driver)
+if (typeof globalThis.crypto === 'undefined') {
+  globalThis.crypto = require('crypto');
+}
+
 // BackOne Proxy Server - Entry Point
 // ─────────────────────────────────────────────────────────────────────────────
 
 const path = require('path');
-require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
+const envFile = process.env.NODE_ENV === 'production' ? '.env.production' : '.env.local';
+require('dotenv').config({ path: path.join(__dirname, '..', envFile) });
 
 const express   = require('express');
 const cors      = require('cors');
@@ -57,11 +63,13 @@ async function main() {
   console.log('╚════════════════════════════════════════════════╝\n');
   console.log(`[Proxy] Mode: ${process.env.PROXY_COLLECT_MODE || 'all'}`);
   
-  // Start REST API server first so liveness probes remain active
+  // Bind to 127.0.0.1 in production — port 4000 must never be exposed externally
-  app.listen(PORT, '0.0.0.0', () => {
+  const BIND_HOST = process.env.NODE_ENV === 'production' ? '127.0.0.1' : '0.0.0.0';
-    console.log(`\n🚀 Proxy REST API running at http://0.0.0.0:${PORT}`);
+  app.listen(PORT, BIND_HOST, () => {
+    console.log(`\n🚀 Proxy REST API running at http://${BIND_HOST}:${PORT}`);
     console.log(`   GET  /health   → liveness check`);
-    console.log(`   GET  /status   → scheduler + DB status\n`);
+    console.log(`   GET  /status   → scheduler + DB status`);
+    console.log(`🔒 Security   : Bound to ${BIND_HOST} (internal only in production)\n`);
   });
 
   const connected = await connectDB();

proxy/models/Schemas.js

@@ -1,182 +1,199 @@
-// proxy/models/Schemas.js
+// proxy/models/Schemas.js
-// MongoDB schemas shared between the proxy server (write) and backend (read).
+// MongoDB schemas shared between the proxy server (write) and backend (read).
-// Each document is tagged with agent_uuid + site_uuid for tenant isolation.
+// Each document is tagged with agent_uuid + site_uuid for tenant isolation.
-//
+//
-// IMPORTANT: Indexes are set for common query patterns:
+// IMPORTANT: Indexes are set for common query patterns:
-//   - timestamp (for time-range queries)
+//   - timestamp (for time-range queries)
-//   - agent_uuid (for per-tenant filtering)
+//   - agent_uuid (for per-tenant filtering)
-//   - site_uuid (for site-level aggregation)
+//   - site_uuid (for site-level aggregation)
-
+
-const mongoose = require('mongoose');
+const mongoose = require('mongoose');
-
+
-const baseOptions = {
+const baseOptions = {
-  timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
+  timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
-};
+};
-
+
-// ─── Bandwidth Summary (per agent, per collection cycle) ───────────────────────
+// ─── Bandwidth Summary (per agent, per collection cycle) ───────────────────────
-const SummarySchema = new mongoose.Schema({
+const SummarySchema = new mongoose.Schema({
-  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
+  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid:     { type: String, index: true },  // null = global/all agents
+  agent_uuid:     { type: String, index: true },  // null = global/all agents
-  site_uuid:      { type: String, index: true },
+  site_uuid:      { type: String, index: true },
-  bandwidth_down: Number,
+  bandwidth_down: Number,
-  bandwidth_up:   Number,
+  bandwidth_up:   Number,
-  active_flows:   Number,
+  active_flows:   Number,
-  download_speed: Number,
+  download_speed: Number,
-  upload_speed:   Number,
+  upload_speed:   Number,
-  total_devices:  Number,
+  total_devices:  Number,
-  total_threats:  Number,
+  total_threats:  Number,
-  packet_drops:   Number,
+  packet_drops:   Number,
-  peak_flow_rate: Number,
+  peak_flow_rate: Number,
-  cpu_usage:      Number,
+  cpu_usage:      Number,
-  memory_usage:   Number,
+  memory_usage:   Number,
-  queue_depth:    Number,
+  queue_depth:    Number,
-}, baseOptions);
+}, baseOptions);
-
+
-// ─── Top Applications (per agent) ─────────────────────────────────────────────
+// ─── Top Applications (per agent) ─────────────────────────────────────────────
-const AppStatSchema = new mongoose.Schema({
+const AppStatSchema = new mongoose.Schema({
-  timestamp:  { type: Date,   required: true, index: true, expires: '7d' },
+  timestamp:  { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid: { type: String, index: true },
+  agent_uuid: { type: String, index: true },
-  site_uuid:  { type: String, index: true },
+  site_uuid:  { type: String, index: true },
-  app_label:  { type: String, required: true },
+  app_label:  { type: String, required: true },
-  download:   Number,
+  download:   Number,
-  upload:     Number,
+  upload:     Number,
-  flows:      Number,
+  flows:      Number,
-}, baseOptions);
+}, baseOptions);
-
+
-// ─── Protocol Statistics (per agent) ──────────────────────────────────────────
+// ─── Protocol Statistics (per agent) ──────────────────────────────────────────
-const ProtocolStatSchema = new mongoose.Schema({
+const ProtocolStatSchema = new mongoose.Schema({
-  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
+  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid:     { type: String, index: true },
+  agent_uuid:     { type: String, index: true },
-  site_uuid:      { type: String, index: true },
+  site_uuid:      { type: String, index: true },
-  protocol_label: { type: String, required: true },
+  protocol_label: { type: String, required: true },
-  download:       Number,
+  download:       Number,
-  upload:         Number,
+  upload:         Number,
-  flows:          Number,
+  flows:          Number,
-}, baseOptions);
+}, baseOptions);
-
+
-// ─── Discovered Devices (per agent, includes IP + MAC + device info) ───────────
+// ─── Discovered Devices (per agent, includes IP + MAC + device info) ───────────
-const DeviceStatSchema = new mongoose.Schema({
+const DeviceStatSchema = new mongoose.Schema({
-  timestamp:    { type: Date,   required: true, index: true, expires: '7d' },
+  timestamp:    { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid:   { type: String, index: true },
+  agent_uuid:   { type: String, index: true },
-  site_uuid:    { type: String, index: true },
+  site_uuid:    { type: String, index: true },
-  ip_address:   { type: String, required: true, index: true },
+  ip_address:   { type: String, required: true, index: true },
-  mac_address:  { type: String, index: true },
+  mac_address:  { type: String, index: true },
-  device_label: String,
+  device_label: String,
-  device_type:  String,
+  device_type:  String,
-  os_label:     String,
+  os_label:     String,
-  manufacturer: String,
+  manufacturer: String,
-  download:     Number,
+  download:     Number,
-  upload:       Number,
+  upload:       Number,
-  flows:        Number,
+  flows:        Number,
-  last_seen:    String,
+  last_seen:    String,
-}, baseOptions);
+}, baseOptions);
-
+
-// ─── Network Flows (per agent) ─────────────────────────────────────────────────
+// ─── Network Flows (per agent) ─────────────────────────────────────────────────
-const FlowSchema = new mongoose.Schema({
+const FlowSchema = new mongoose.Schema({
-  timestamp:  { type: Date,   required: true, index: true, expires: '7d' },
+  timestamp:  { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid: { type: String, index: true },
+  agent_uuid: { type: String, index: true },
-  site_uuid:  { type: String, index: true },
+  site_uuid:  { type: String, index: true },
-  flow_id:    String,
+  flow_id:    String,
-  src_ip:     { type: String, index: true },
+  src_ip:     { type: String, index: true },
-  src_mac:    { type: String, index: true },  // indexed for MAC-to-IP resolution in events
+  src_mac:    { type: String, index: true },  // indexed for MAC-to-IP resolution in events
-  dst_ip:     { type: String, index: true },
+  dst_ip:     { type: String, index: true },
-  dst_port:   Number,
+  dst_port:   Number,
-  protocol:   String,
+  protocol:   String,
-  app_label:  String,
+  app_label:  String,
-  domain:     String,
+  domain:     String,
-  download:   Number,
+  download:   Number,
-  upload:     Number,
+  upload:     Number,
-  first_seen: String,
+  first_seen: String,
-  last_seen:  String,
+  last_seen:  String,
-}, baseOptions);
+}, baseOptions);
-
+
-// ─── Cyber Threats (per agent) ─────────────────────────────────────────────────
+// ─── Cyber Threats (per agent) ─────────────────────────────────────────────────
-const ThreatSchema = new mongoose.Schema({
+const ThreatSchema = new mongoose.Schema({
-  timestamp:   { type: Date,   required: true, index: true, expires: '7d' },
+  timestamp:   { type: Date,   required: true, index: true, expires: '7d' },
-  agent_uuid:  { type: String, index: true },
+  agent_uuid:  { type: String, index: true },
-  site_uuid:   { type: String, index: true },
+  site_uuid:   { type: String, index: true },
-  threat_type: String,
+  threat_type: String,
-  severity:    String,
+  severity:    String,
-  src_ip:      String,
+  src_ip:      String,
-  dst_ip:      String,
+  dst_ip:      String,
-  dst_port:    Number,
+  dst_port:    Number,
-  protocol:    String,
+  protocol:    String,
-  description: String,
+  description: String,
-  event_at:    String,
+  event_at:    String,
-}, baseOptions);
+  flow_id:     { type: String, index: true },
-
+}, baseOptions);
-// ─── App Categories (per agent) ───────────────────────────────────────────────
+
-const AppCategoryStatSchema = new mongoose.Schema({
+// ─── App Categories (per agent) ───────────────────────────────────────────────
-  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
+const AppCategoryStatSchema = new mongoose.Schema({
-  agent_uuid:     { type: String, index: true },
+  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
-  site_uuid:      { type: String, index: true },
+  agent_uuid:     { type: String, index: true },
-  category_label: { type: String, required: true },
+  site_uuid:      { type: String, index: true },
-  download:       Number,
+  category_label: { type: String, required: true },
-  upload:         Number,
+  download:       Number,
-  flows:          Number,
+  upload:         Number,
-}, baseOptions);
+  flows:          Number,
-
+}, baseOptions);
-// ─── System Events (per agent) ─────────────────────────────────────────────────
+
-const EventSchema = new mongoose.Schema({
+// ─── System Events (per agent) ─────────────────────────────────────────────────
-  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
+const EventSchema = new mongoose.Schema({
-  agent_uuid:     { type: String, index: true },
+  timestamp:      { type: Date,   required: true, index: true, expires: '7d' },
-  site_uuid:      { type: String, index: true },
+  agent_uuid:     { type: String, index: true },
-  event_id:       Number,
+  site_uuid:      { type: String, index: true },
-  event_type:     String,
+  event_id:       Number,
-  severity:       String,
+  event_type:     String,
-  description:    String,
+  severity:       String,
-  category_label: String,
+  description:    String,
-  ip_address:     String,
+  category_label: String,
-  mac_address:    String,
+  ip_address:     String,
-  event_at:       Date,
+  mac_address:    String,
-}, baseOptions);
+  event_at:       Date,
-
+  flow_id:        { type: String, index: true },
-// ─── Compound indexes for common dashboard queries ─────────────────────────────
+}, baseOptions);
-SummarySchema.index({ agent_uuid: 1, timestamp: -1 });
+
-AppStatSchema.index({ agent_uuid: 1, timestamp: -1, download: -1 });
+// ─── Compound indexes for common dashboard queries ─────────────────────────────
-DeviceStatSchema.index({ agent_uuid: 1, ip_address: 1 }, { unique: true });
+SummarySchema.index({ agent_uuid: 1, timestamp: -1 });
-FlowSchema.index({ agent_uuid: 1, timestamp: -1 });
+AppStatSchema.index({ agent_uuid: 1, timestamp: -1, download: -1 });
-FlowSchema.index({ agent_uuid: 1, flow_id: 1 });
+DeviceStatSchema.index({ agent_uuid: 1, ip_address: 1 }, { unique: true });
-FlowSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
+FlowSchema.index({ agent_uuid: 1, timestamp: -1 });
-ThreatSchema.index({ agent_uuid: 1, timestamp: -1 });
+FlowSchema.index({ agent_uuid: 1, flow_id: 1 });
-AppCategoryStatSchema.index({ agent_uuid: 1, timestamp: -1 });
+FlowSchema.index({ site_uuid: 1, timestamp: -1 });
-EventSchema.index({ agent_uuid: 1, timestamp: -1 });
+FlowSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
-FlowSchema.index({ site_uuid: 1, src_mac: 1, timestamp: -1 }); // for MAC-to-IP resolution
+ThreatSchema.index({ agent_uuid: 1, timestamp: -1 });
-
+AppCategoryStatSchema.index({ agent_uuid: 1, timestamp: -1 });
-// ── Per-Device Per-Application Stats ────────────────────────────────────────
+EventSchema.index({ agent_uuid: 1, timestamp: -1 });
-// Collected from DPI API: /data/stats/top/application/download with filter_local_ips
+FlowSchema.index({ site_uuid: 1, src_mac: 1, timestamp: -1 }); // for MAC-to-IP resolution
-// Allows showing "YouTube 134GB" in Device Detail modal per specific IP
+
-const DeviceAppStatSchema = new mongoose.Schema({
+// ── Per-Device Per-Application Stats ────────────────────────────────────────
-  timestamp:  { type: Date,   required: true, index: true, expires: '7d' },
+// Collected from DPI API: /data/stats/top/application/download with filter_local_ips
-  agent_uuid: { type: String, index: true },
+// Allows showing "YouTube 134GB" in Device Detail modal per specific IP
-  site_uuid:  { type: String, index: true },
+const DeviceAppStatSchema = new mongoose.Schema({
-  ip_address: { type: String, required: true, index: true },
+  timestamp:  { type: Date,   required: true, index: true, expires: '7d' },
-  app_label:  { type: String, required: true },
+  agent_uuid: { type: String, index: true },
-  app_id:     Number,
+  site_uuid:  { type: String, index: true },
-  download:   { type: Number, default: 0 },
+  ip_address: { type: String, required: true, index: true },
-  upload:     { type: Number, default: 0 },
+  app_label:  { type: String, required: true },
-  flows:      { type: Number, default: 0 },
+  app_id:     Number,
-  last_seen:  String,
+  download:   { type: Number, default: 0 },
-}, baseOptions);
+  upload:     { type: Number, default: 0 },
-DeviceAppStatSchema.index({ agent_uuid: 1, ip_address: 1, timestamp: -1 });
+  flows:      { type: Number, default: 0 },
-DeviceAppStatSchema.index({ ip_address: 1, app_label: 1, timestamp: -1 });
+  last_seen:  String,
-DeviceAppStatSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
+}, baseOptions);
-
+DeviceAppStatSchema.index({ agent_uuid: 1, ip_address: 1, timestamp: -1 });
-const telemetrySchemas = require('./SchemasTelemetry');
+DeviceAppStatSchema.index({ ip_address: 1, app_label: 1, timestamp: -1 });
-const auxSchemas = require('./SchemasAux');
+DeviceAppStatSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
-
+
-module.exports = {
+const telemetrySchemas = require('./SchemasTelemetry');
-  Summary:     mongoose.model('Summary',     SummarySchema),
+const auxSchemas = require('./SchemasAux');
-  AppStat:     mongoose.model('AppStat',     AppStatSchema),
+
-  ProtocolStat:mongoose.model('ProtocolStat',ProtocolStatSchema),
+// ─── Agent Registry (all agents registered in Netify, regardless of activity) ──
-  DeviceStat:  mongoose.model('DeviceStat',  DeviceStatSchema),
+// Upserted every collector cycle. Source of truth for the agents list page.
-  DeviceAppStat: mongoose.model('DeviceAppStat', DeviceAppStatSchema),
+const AgentRegistrySchema = new mongoose.Schema({
-  Flow:        mongoose.model('Flow',        FlowSchema),
+  uuid:         { type: String, required: true, unique: true, index: true },
-  Threat:      mongoose.model('Threat',      ThreatSchema),
+  serial:       { type: String },
-  AppCategoryStat: mongoose.model('AppCategoryStat', AppCategoryStatSchema),
+  label:        { type: String },
-  Event:         mongoose.model('Event',         EventSchema),
+  site_uuid:    { type: String, index: true },
-  ...auxSchemas,
+  provisioned:  { type: Boolean, default: false },
-  ...telemetrySchemas,
+  activated:    { type: Boolean, default: false },
-};
+  last_seen_at: { type: mongoose.Schema.Types.Mixed },
-
+  netify_id:    { type: Number },
-
+}, { ...baseOptions, collection: 'agent_registry' });
+
+module.exports = {
+  Summary:       mongoose.model('Summary',       SummarySchema),
+  AppStat:       mongoose.model('AppStat',       AppStatSchema),
+  ProtocolStat:  mongoose.model('ProtocolStat',  ProtocolStatSchema),
+  DeviceStat:    mongoose.model('DeviceStat',    DeviceStatSchema),
+  DeviceAppStat: mongoose.model('DeviceAppStat', DeviceAppStatSchema),
+  Flow:          mongoose.model('Flow',          FlowSchema),
+  Threat:        mongoose.model('Threat',        ThreatSchema),
+  AppCategoryStat: mongoose.model('AppCategoryStat', AppCategoryStatSchema),
+  Event:         mongoose.model('Event',         EventSchema),
+  AgentRegistry: mongoose.model('AgentRegistry', AgentRegistrySchema),
+  ...auxSchemas,
+  ...telemetrySchemas,
+};
+
+

proxy/netifyAgentFetcher.js

@@ -0,0 +1,89 @@
+// proxy/netifyAgentFetcher.js
+// ─────────────────────────────────────────────────────────────────────────────
+// Fetches active agents from Netify Informatics API.
+// Uses a two-strategy approach to handle endpoints that may timeout (Source 2).
+// ─────────────────────────────────────────────────────────────────────────────
+
+const { netifyFetch, agentMap } = require('./netifyClientCore');
+
+// Strategy 1 timeout: 15s (agent/download can be slow on small deployments)
+const PRIMARY_TIMEOUT_MS = 15000;
+
+async function fetchAgents(siteUuid = null) {
+  // Strategy 1: Use /data/stats/top/agent/download (standard Netify endpoint)
+  // filter_interval reduced to 31 days to lessen query load vs. old 365-day value.
+  const primaryPromise = (async () => {
+    try {
+      const data = await netifyFetch('/data/stats/top/agent/download', {
+        filter_interval: 44640, // 31 days
+        settings_limit: 1000000,
+      }, null, siteUuid);
+      if (data && Array.isArray(data) && data.length > 0) return data;
+      return null;
+    } catch (e) {
+      return null;
+    }
+  })();
+
+  const timeoutPromise = new Promise(resolve =>
+    setTimeout(() => resolve(null), PRIMARY_TIMEOUT_MS)
+  );
+
+  const primaryData = await Promise.race([primaryPromise, timeoutPromise]);
+
+  if (primaryData && Array.isArray(primaryData) && primaryData.length > 0) {
+    const list = primaryData.map(r => ({
+      id:           r.agent?.id,
+      uuid:         r.agent?.uuid || r.agent?.serial,
+      serial:       r.agent?.serial,
+      label:        r.agent?.label || r.agent?.serial,
+      provisioned:  true,
+      activated:    true,
+      last_seen_at: r.agent?.last_seen_at ?? null,
+    })).filter(a => a.uuid);
+
+    // Populate agentMap (uuid → id) for downstream filter_agents usage
+    for (const a of list) {
+      if (a.uuid && a.id) agentMap[a.uuid] = a.id;
+    }
+    return list;
+  }
+
+  // Strategy 2: Fallback — discover agents from /data/flows
+  // Useful for Source 2 where /data/stats/top/agent/download consistently times out.
+  console.log('[fetchAgents] Primary endpoint timeout/empty. Using flows-based agent discovery...');
+  try {
+    const flowData = await netifyFetch('/data/flows', {
+      settings_limit: 100,
+    }, null, siteUuid);
+
+    if (!flowData || !Array.isArray(flowData)) return [];
+
+    // Extract unique agent_uuids from flow records
+    const seen = new Set();
+    const agentList = [];
+    for (const flow of flowData) {
+      const uuid = flow.agent_uuid;
+      if (uuid && !seen.has(uuid)) {
+        seen.add(uuid);
+        agentList.push({
+          id:           null,
+          uuid:         uuid,
+          serial:       uuid,
+          label:        uuid,
+          provisioned:  true,
+          activated:    true,
+          last_seen_at: flow.last_seen_at ?? null,
+        });
+      }
+    }
+
+    console.log(`[fetchAgents] Fallback discovered ${agentList.length} agent(s) from flows.`);
+    return agentList;
+  } catch (e) {
+    console.error('[fetchAgents] Fallback also failed:', e.message);
+    return [];
+  }
+}
+
+module.exports = { fetchAgents };

proxy/netifyClient.js

@@ -3,157 +3,11 @@
 // DPI API wrapper for the BackOne Proxy Server targeting original Netify API.
 // ─────────────────────────────────────────────────────────────────────────────
 
-const { netifyFetch, BASE_URL, agentMap } = require('./netifyClientCore');
+const { netifyFetch, BASE_URL } = require('./netifyClientCore');
 const telemetry = require('./netifyTelemetry');
+const stats = require('./netifyClientStats');
 
-const PORT_SERVICE_MAP = {
+async function fetchFlows(limit = 1000000, agentUuid = null, siteUuid = null) {
-  80: 'HTTP', 443: 'HTTPS / TLS', 8080: 'HTTP Alt', 8443: 'HTTPS Alt',
-  53: 'DNS', 5353: 'mDNS', 853: 'DNS-over-TLS',
-  25: 'SMTP', 587: 'SMTP TLS', 465: 'SMTPS', 110: 'POP3', 143: 'IMAP',
-  22: 'SSH', 23: 'Telnet', 3389: 'RDP', 5900: 'VNC',
-  21: 'FTP', 20: 'FTP Data', 989: 'FTPS', 990: 'FTPS Control',
-  3306: 'MySQL', 5432: 'PostgreSQL', 6379: 'Redis', 27017: 'MongoDB',
-  1194: 'OpenVPN', 51820: 'WireGuard', 500: 'IPSec IKE', 4500: 'IPSec NAT-T',
-  67: 'DHCP', 68: 'DHCP Client', 123: 'NTP',
-  6881: 'BitTorrent', 6882: 'BitTorrent', 6883: 'BitTorrent',
-  9993: 'ZeroTier VPN',
-};
-
-async function fetchAgents(siteUuid = null) {
-  const data = await netifyFetch('/data/stats/top/agent/download', { filter_interval: 43200, settings_limit: 100 }, null, siteUuid);
-  if (!data || !Array.isArray(data)) return [];
-  const list = data.map(r => ({
-    id:    r.agent?.id,
-    uuid:  r.agent?.uuid,
-    label: r.agent?.label,
-  })).filter(a => a.uuid);
-
-  for (const a of list) {
-    if (a.uuid && a.id) agentMap[a.uuid] = a.id;
-  }
-
-  // Secondary validation: if this site already has data in MongoDB, only return agents
-  // that have at least one summary record for THIS site_uuid. This prevents the
-  // org-level stats endpoint from cross-contaminating agents across sites.
-  if (siteUuid) {
-    try {
-      const mongoose = require('mongoose');
-      if (mongoose.connection.readyState === 1) {
-        const db = mongoose.connection.db;
-        const knownAgents = await db.collection('summaries').distinct('agent_uuid', {
-          site_uuid:  siteUuid,
-          agent_uuid: { $ne: null },
-        });
-
-        if (knownAgents.length > 0) {
-          const knownSet = new Set(knownAgents);
-          const validated = list.filter(a => knownSet.has(a.uuid));
-          // If MongoDB cross-check yields results, use the validated list.
-          // On first boot (no DB data yet), fall through and use the full API list.
-          if (validated.length > 0) return validated;
-        }
-      }
-    } catch (err) {
-      console.warn('[Collector] fetchAgents DB cross-check failed:', err.message);
-    }
-  }
-
-  return list;
-}
-
-
-async function fetchBandwidthSummary(interval = 1440, agentUuid = null, siteUuid = null) {
-  const [dlData, ulData, flowsData] = await Promise.all([
-    netifyFetch('/data/stats/top/local_ip/download', { filter_interval: interval, settings_limit: 500 }, agentUuid, siteUuid),
-    netifyFetch('/data/stats/top/local_ip/upload', { filter_interval: interval, settings_limit: 500 }, agentUuid, siteUuid),
-    netifyFetch('/data/stats/top/local_ip/flow_count', { filter_interval: interval, settings_limit: 500 }, agentUuid, siteUuid),
-  ]);
-  const bandwidth_down = dlData?.reduce((s, r) => s + (r.download ?? 0), 0) ?? 0;
-  const bandwidth_up   = ulData?.reduce((s, r) => s + (r.upload ?? 0), 0) ?? 0;
-  const total_devices  = dlData?.length ?? 0;
-  const active_flows   = flowsData?.reduce((s, r) => s + (r.flows ?? r.flow_count ?? 0), 0) ?? 0;
-  return { bandwidth_down, bandwidth_up, total_devices, active_flows, total_threats: 0 };
-}
-
-async function fetchTopApps(interval = 1440, limit = 200, agentUuid = null, siteUuid = null) {
-  const [dlData, ulData] = await Promise.all([
-    netifyFetch('/data/stats/top/application/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
-    netifyFetch('/data/stats/top/application/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
-  ]);
-  if (!dlData) return null;
-  const ulMap = {};
-  if (ulData) {
-    for (const r of ulData) {
-      const id = r.application?.id;
-      if (id) ulMap[id] = r.upload ?? 0;
-    }
-  }
-  return dlData.map(r => ({
-    application: {
-      id:    r.application?.id ?? null,
-      label: r.application?.label ?? 'Unknown',
-      tag:   r.application?.tag ?? null,
-    },
-    download: r.download ?? 0,
-    upload:   ulMap[r.application?.id] ?? 0,
-    flows:    r.flows ?? 0,
-  }));
-}
-
-async function fetchDiscoveredDevices(interval = 1440, limit = 500, agentUuid = null, siteUuid = null) {
-  const [dlData, ulData] = await Promise.all([
-    netifyFetch('/data/stats/top/local_ip/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
-    netifyFetch('/data/stats/top/local_ip/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
-  ]);
-  if (!dlData) return null;
-  const ulMap = {};
-  if (ulData) {
-    for (const r of ulData) {
-      const ip = r.local_ip?.address ?? String(r.local_ip);
-      ulMap[ip] = r.upload ?? 0;
-    }
-  }
-  return dlData.map(r => {
-    const ip = r.local_ip?.address ?? String(r.local_ip ?? '');
-    return {
-      ip_address:   ip,
-      mac_address:  r.local_mac ?? null,
-      device_label: r.device_label ?? ip,
-      device_type:  r.device_type ?? null,
-      os_label:     r.os_label ?? null,
-      manufacturer: r.manufacturer ?? null,
-      download:     r.download ?? 0,
-      upload:       ulMap[ip] ?? 0,
-      flows:        r.flows ?? 0,
-      last_seen:    r.last_seen_at?.date ?? null,
-    };
-  }).filter(d => d.ip_address);
-}
-
-async function fetchDeviceApps(ipAddress, interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
-  const ipFilter = JSON.stringify([ipAddress]);
-  const [dlData, ulData] = await Promise.all([
-    netifyFetch('/data/stats/top/application/download', { filter_interval: interval, settings_limit: limit, filter_local_ips: ipFilter }, agentUuid, siteUuid),
-    netifyFetch('/data/stats/top/application/upload', { filter_interval: interval, settings_limit: limit, filter_local_ips: ipFilter }, agentUuid, siteUuid),
-  ]);
-  if (!dlData || !Array.isArray(dlData)) return [];
-  const ulMap = {};
-  if (ulData && Array.isArray(ulData)) {
-    for (const r of ulData) {
-      const id = r.application?.id;
-      if (id) ulMap[id] = r.upload ?? 0;
-    }
-  }
-  return dlData.map(r => ({
-    app_label: r.application?.label ?? 'Unknown',
-    app_id:    r.application?.id    ?? null,
-    download:  r.download ?? 0,
-    upload:    ulMap[r.application?.id] ?? 0,
-    flows:     r.flows ?? 0,
-  })).filter(a => a.download > 0 || a.upload > 0);
-}
-
-async function fetchFlows(limit = 10000, agentUuid = null, siteUuid = null) {
   const [raw, sniRaw] = await Promise.all([
     netifyFetch('/data/flows', { settings_limit: limit }, agentUuid, siteUuid),
     netifyFetch('/data/stats/top/tls_sni/download', { filter_interval: 1440, settings_limit: 50 }, agentUuid, siteUuid),
@@ -163,12 +17,14 @@ async function fetchFlows(limit = 10000, agentUuid = null, siteUuid = null) {
   if (sniRaw && Array.isArray(sniRaw)) {
     for (const r of sniRaw) {
       const sni = typeof r.tls_sni === 'object' ? r.tls_sni?.label : r.tls_sni;
-      if (sni && typeof sni === 'string' && sni.trim() !== '') sniList.push(sni.replace(/^\*\./, '').trim());
+      if (sni && typeof sni === 'string' && sni.trim() !== '') {
+        sniList.push(sni.replace(/^\*\./, '').trim());
+      }
     }
   }
   return raw.map(r => {
     const port = r.remote_port ?? null;
-    const portService = port ? (PORT_SERVICE_MAP[port] ?? `Port ${port}`) : null;
+    const portService = port ? (stats.PORT_SERVICE_MAP[port] ?? `Port ${port}`) : null;
     const appLabel = r.application?.label || portService;
     const domain = r.tls_sni || r.dns_hostname || r.hostname || null;
     return {
@@ -188,84 +44,17 @@ async function fetchFlows(limit = 10000, agentUuid = null, siteUuid = null) {
   }).filter(f => f.src_ip);
 }
 
-async function fetchCyberThreats(agentUuid = null, siteUuid = null) {
-  const ipRepData = await netifyFetch('/data/stats/top/remote_ip/download', { filter_interval: 1440, settings_limit: 50 }, agentUuid, siteUuid);
-  if (!ipRepData || !Array.isArray(ipRepData)) return [];
-  const SUSPICIOUS_PORTS = new Set([23, 4444, 1337, 6667, 31337, 12345, 54321, 4899, 5554, 9999]);
-  const threats = [];
-  for (const r of ipRepData) {
-    const ip   = r.remote_ip?.address ?? null;
-    const port = r.remote_port ?? 0;
-    if (ip && SUSPICIOUS_PORTS.has(port)) {
-      threats.push({
-        threat_type: `Suspicious Port ${port}`,
-        severity:    'High',
-        src_ip:      null,
-        dst_ip:      ip,
-        dst_port:    port,
-        protocol:    r.ip_protocol?.label ?? null,
-        description: `Suspicious outbound connection to ${ip}:${port}`,
-        event_at:    new Date().toISOString(),
-      });
-    }
-  }
-  return threats;
-}
-
-async function fetchEvents(limit = 100, agentUuid = null, siteUuid = null) {
-  const raw = await netifyFetch('/event/events', { settings_limit: limit }, agentUuid, siteUuid);
-  if (!raw || !Array.isArray(raw)) return [];
-  return raw.map(r => {
-    let msg = r.label || '';
-    if (r.description) {
-      try {
-        const descObj = JSON.parse(r.description);
-        msg = descObj.default || r.label || '';
-        if (descObj.tags) {
-          for (const k in descObj.tags) {
-            const tagVal = descObj.tags[k];
-            const val = Array.isArray(tagVal) ? (tagVal[0] === 'Unknown' && tagVal[1] ? tagVal[1] : tagVal[0]) : tagVal;
-            msg = msg.replace(`{{ ${k} }}`, val).replace(`{{${k}}}`, val);
-          }
-        }
-      } catch (e) {
-        msg = r.description;
-      }
-    }
-    let sevLabel = 'Info';
-    if (r.severity >= 30) sevLabel = 'Critical';
-    else if (r.severity >= 20) sevLabel = 'High';
-    else if (r.severity >= 10) sevLabel = 'Warning';
-    let srcIp = null;
-    if (r.description) {
-      try {
-        const descObj = JSON.parse(r.description);
-        srcIp = descObj.tags?.device_ip || descObj.tags?.ip || null;
-      } catch {}
-    }
-    return {
-      event_id:       r.id || null,
-      event_type:     r.basename || 'unknown',
-      severity:       sevLabel,
-      description:    msg,
-      category_label: r.category?.label || 'Intelligence',
-      ip_address:     srcIp,
-      mac_address:    r.additional?.device?.mac?.address || null,
-      event_at:       r.created_at?.date ? new Date(r.created_at.date) : new Date()
-    };
-  });
-}
-
 module.exports = {
-  fetchAgents,
-  fetchBandwidthSummary,
-  fetchTopApps,
-  fetchDiscoveredDevices,
-  fetchDeviceApps,
   fetchFlows,
-  fetchCyberThreats,
+  fetchAgents: stats.fetchAgents,
-  fetchEvents,
+  fetchBandwidthSummary: stats.fetchBandwidthSummary,
+  fetchTopApps: stats.fetchTopApps,
+  fetchDiscoveredDevices: stats.fetchDiscoveredDevices,
+  fetchDeviceApps: stats.fetchDeviceApps,
+  fetchCyberThreats: stats.fetchCyberThreats,
+  fetchEvents: stats.fetchEvents,
+  syncApplicationDictionary: stats.syncApplicationDictionary,
   BASE_URL,
-  PORT_SERVICE_MAP,
+  PORT_SERVICE_MAP: stats.PORT_SERVICE_MAP,
   ...telemetry,
 };

proxy/netifyClientStats.js

@@ -0,0 +1,219 @@
+// proxy/netifyClientStats.js
+// ─────────────────────────────────────────────────────────────────────────────
+// Supplementary fetchers split from netifyClient.js to satisfy the 256-line limit.
+// ─────────────────────────────────────────────────────────────────────────────
+
+const { netifyFetch } = require('./netifyClientCore');
+const { fetchAgents } = require('./netifyAgentFetcher');
+
+const PORT_SERVICE_MAP = {
+  80: 'HTTP', 443: 'HTTPS / TLS', 8080: 'HTTP Alt', 8443: 'HTTPS Alt',
+  53: 'DNS', 5353: 'mDNS', 853: 'DNS-over-TLS',
+  25: 'SMTP', 587: 'SMTP TLS', 465: 'SMTPS', 110: 'POP3', 143: 'IMAP',
+  22: 'SSH', 23: 'Telnet', 3389: 'RDP', 5900: 'VNC',
+  21: 'FTP', 20: 'FTP Data', 989: 'FTPS', 990: 'FTPS Control',
+  3306: 'MySQL', 5432: 'PostgreSQL', 6379: 'Redis', 27017: 'MongoDB',
+  1194: 'OpenVPN', 51820: 'WireGuard', 500: 'IPSec IKE', 4500: 'IPSec NAT-T',
+  67: 'DHCP', 68: 'DHCP Client', 123: 'NTP',
+  6881: 'BitTorrent', 6882: 'BitTorrent', 6883: 'BitTorrent',
+  9993: 'ZeroTier VPN',
+};
+
+async function fetchBandwidthSummary(interval = 1440, agentUuid = null, siteUuid = null) {
+  const [dlData, ulData, flowsData] = await Promise.all([
+    netifyFetch('/data/stats/top/local_ip/download', { filter_interval: interval, settings_limit: 500 }, agentUuid, siteUuid),
+    netifyFetch('/data/stats/top/local_ip/upload', { filter_interval: interval, settings_limit: 500 }, agentUuid, siteUuid),
+    netifyFetch('/data/stats/top/local_ip/flow_count', { filter_interval: interval, settings_limit: 500 }, agentUuid, siteUuid),
+  ]);
+  const bandwidth_down = dlData?.reduce((s, r) => s + (r.download ?? 0), 0) ?? 0;
+  const bandwidth_up   = ulData?.reduce((s, r) => s + (r.upload ?? 0), 0) ?? 0;
+  const total_devices  = dlData?.length ?? 0;
+  const active_flows   = flowsData?.reduce((s, r) => s + (r.flows ?? r.flow_count ?? 0), 0) ?? 0;
+  return { bandwidth_down, bandwidth_up, total_devices, active_flows, total_threats: 0 };
+}
+
+async function fetchTopApps(interval = 1440, limit = 200, agentUuid = null, siteUuid = null) {
+  const [dlData, ulData] = await Promise.all([
+    netifyFetch('/data/stats/top/application/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    netifyFetch('/data/stats/top/application/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+  ]);
+  if (!dlData) return null;
+  const ulMap = {};
+  if (ulData) {
+    for (const r of ulData) {
+      const id = r.application?.id;
+      if (id) ulMap[id] = r.upload ?? 0;
+    }
+  }
+  return dlData.map(r => ({
+    application: { id: r.application?.id ?? null, label: r.application?.label ?? 'Unknown', tag: r.application?.tag ?? null },
+    download: r.download ?? 0, upload: ulMap[r.application?.id] ?? 0, flows: r.flows ?? 0,
+  }));
+}
+
+async function fetchDiscoveredDevices(interval = 1440, limit = 500, agentUuid = null, siteUuid = null) {
+  const [dlData, ulData] = await Promise.all([
+    netifyFetch('/data/stats/top/local_ip/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    netifyFetch('/data/stats/top/local_ip/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+  ]);
+  if (!dlData) return null;
+  const ulMap = {};
+  if (ulData) {
+    for (const r of ulData) {
+      const ip = r.local_ip?.address ?? String(r.local_ip);
+      ulMap[ip] = r.upload ?? 0;
+    }
+  }
+  return dlData.map(r => {
+    const ip = r.local_ip?.address ?? String(r.local_ip ?? '');
+    return {
+      ip_address: ip, mac_address: r.local_mac ?? null, device_label: r.device_label ?? ip, device_type: r.device_type ?? null,
+      os_label: r.os_label ?? null, manufacturer: r.manufacturer ?? null, download: r.download ?? 0, upload: ulMap[ip] ?? 0,
+      flows: r.flows ?? 0, last_seen: r.last_seen_at?.date ?? null,
+    };
+  }).filter(d => d.ip_address);
+}
+
+async function fetchDeviceApps(ipAddress, interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+  const ipFilter = JSON.stringify([ipAddress]);
+  const [dlData, ulData] = await Promise.all([
+    netifyFetch('/data/stats/top/application/download', { filter_interval: interval, settings_limit: limit, filter_local_ips: ipFilter }, agentUuid, siteUuid),
+    netifyFetch('/data/stats/top/application/upload', { filter_interval: interval, settings_limit: limit, filter_local_ips: ipFilter }, agentUuid, siteUuid),
+  ]);
+  if (!dlData || !Array.isArray(dlData)) return [];
+  const ulMap = {};
+  if (ulData && Array.isArray(ulData)) {
+    for (const r of ulData) {
+      const id = r.application?.id;
+      if (id) ulMap[id] = r.upload ?? 0;
+    }
+  }
+  return dlData.map(r => ({
+    app_label: r.application?.label ?? 'Unknown',
+    app_id:    r.application?.id    ?? null,
+    download:  r.download ?? 0,
+    upload:    ulMap[r.application?.id] ?? 0,
+    flows:     r.flows ?? 0,
+  })).filter(a => a.download > 0 || a.upload > 0);
+}
+
+async function fetchCyberThreats(agentUuid = null, siteUuid = null) {
+  const ipRepData = await netifyFetch('/data/stats/top/remote_ip/download', { filter_interval: 1440, settings_limit: 50 }, agentUuid, siteUuid);
+  if (!ipRepData || !Array.isArray(ipRepData)) return [];
+  const SUSPICIOUS_PORTS = new Set([23, 4444, 1337, 6667, 31337, 12345, 54321, 4899, 5554, 9999]);
+  const threats = [];
+  for (const r of ipRepData) {
+    const ip   = r.remote_ip?.address ?? null;
+    const port = r.remote_port ?? 0;
+    if (ip && SUSPICIOUS_PORTS.has(port)) {
+      threats.push({
+        threat_type: `Suspicious Port ${port}`,
+        severity:    'High',
+        src_ip:      null,
+        dst_ip:      ip,
+        dst_port:    port,
+        protocol:    r.ip_protocol?.label ?? null,
+        description: `Suspicious outbound connection to ${ip}:${port}`,
+        event_at:    new Date().toISOString(),
+      });
+    }
+  }
+  return threats;
+}
+
+async function fetchEvents(limit = 100, agentUuid = null, siteUuid = null) {
+  const raw = await netifyFetch('/event/events', { settings_limit: limit }, agentUuid, siteUuid);
+  if (!raw || !Array.isArray(raw)) return [];
+  return raw.map(r => {
+    let msg = r.label || '';
+    if (r.description) {
+      try {
+        const descObj = JSON.parse(r.description);
+        msg = descObj.default || r.label || '';
+        if (descObj.tags) {
+          for (const k in descObj.tags) {
+            const tagVal = descObj.tags[k];
+            const val = Array.isArray(tagVal) ? (tagVal[0] === 'Unknown' && tagVal[1] ? tagVal[1] : tagVal[0]) : tagVal;
+            msg = msg.replace(`{{ ${k} }}`, val).replace(`{{${k}}}`, val);
+          }
+        }
+      } catch (e) {
+        msg = r.description;
+      }
+    }
+    let sevLabel = 'Info';
+    if (r.severity >= 30) sevLabel = 'Critical';
+    else if (r.severity >= 20) sevLabel = 'High';
+    else if (r.severity >= 10) sevLabel = 'Warning';
+    let srcIp = null;
+    if (r.description) {
+      try {
+        const descObj = JSON.parse(r.description);
+        srcIp = descObj.tags?.device_ip || descObj.tags?.ip || null;
+      } catch {}
+    }
+    return {
+      event_id:       r.id || null,
+      event_type:     r.basename || 'unknown',
+      severity:       sevLabel,
+      description:    msg,
+      category_label: r.category?.label || 'Intelligence',
+      ip_address:     srcIp,
+      mac_address:    r.additional?.device?.mac?.address || null,
+      event_at:       r.created_at?.date ? new Date(r.created_at.date) : new Date()
+    };
+  });
+}
+
+async function syncApplicationDictionary() {
+  const mongoose = require('mongoose');
+  const { LookupApp } = require('./models/Schemas');
+  
+  console.log('[Netify] Fetching application catalog...');
+  const allApps = await netifyFetch('/lookup/applications', { settings_limit: 5000 });
+  if (!allApps || !Array.isArray(allApps)) {
+    console.error('[Netify] Failed to fetch application dictionary.');
+    return;
+  }
+  
+  console.log(`[Netify] Application catalog fetched successfully. Got ${allApps.length} apps.`);
+  if (allApps.length === 0) return;
+  
+  console.log(`[Netify] Syncing ${allApps.length} application definitions to MongoDB...`);
+  await LookupApp.deleteMany({});
+  
+  const batchSize = 100;
+  for (let i = 0; i < allApps.length; i += batchSize) {
+    const batch = allApps.slice(i, i + batchSize);
+    await LookupApp.insertMany(batch.map(app => ({
+      id: app.id,
+      name: app.name,
+      label: app.label,
+      tag: app.tag,
+      description: app.description,
+      full_name: app.full_name || app.application?.full_label || null,
+      favicon: app.favicon || app.application?.favicon || null,
+      icon: app.icon || app.application?.icon || null,
+      logo: app.logo || app.application?.logo || null,
+      application_category: {
+        id: app.application_category?.id,
+        name: app.application_category?.name,
+        label: app.application_category?.label,
+        tag: app.application_category?.tag
+      }
+    })));
+  }
+  console.log('[Netify] ✓ Application dictionary sync completed.');
+}
+
+module.exports = {
+  fetchAgents,
+  fetchBandwidthSummary,
+  fetchTopApps,
+  fetchDiscoveredDevices,
+  fetchDeviceApps,
+  fetchCyberThreats,
+  fetchEvents,
+  syncApplicationDictionary,
+  PORT_SERVICE_MAP
+};

proxy/netifyTelemetry.js

@@ -1,234 +1,234 @@
-// proxy/netifyTelemetry.js
+// proxy/netifyTelemetry.js
-// ─────────────────────────────────────────────────────────────────────────────
+// ─────────────────────────────────────────────────────────────────────────────
-// Supplementary Telemetry endpoints wrapper for BackOne Proxy Server
+// Supplementary Telemetry endpoints wrapper for BackOne Proxy Server
-// Split from netifyClient.js to strictly respect the 256-line file size limit.
+// Split from netifyClient.js to strictly respect the 256-line file size limit.
-// ─────────────────────────────────────────────────────────────────────────────
+// ─────────────────────────────────────────────────────────────────────────────
-
+
-const { netifyFetch } = require('./netifyClientCore');
+const { netifyFetch } = require('./netifyClientCore');
-
+
-async function fetchTopAppCategories(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
+async function fetchTopAppCategories(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
-  const [dlData, ulData] = await Promise.all([
+  const [dlData, ulData] = await Promise.all([
-    netifyFetch('/data/stats/top/application_category/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    netifyFetch('/data/stats/top/application_category/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
-    netifyFetch('/data/stats/top/application_category/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    netifyFetch('/data/stats/top/application_category/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
-  ]);
+  ]);
-  if (!dlData) return [];
+  if (!dlData) return [];
-  const ulMap = {};
+  const ulMap = {};
-  if (ulData) {
+  if (ulData) {
-    for (const r of ulData) {
+    for (const r of ulData) {
-      const key = r.application_category?.label ?? r.application_category;
+      const key = r.application_category?.label ?? r.application_category;
-      if (key) ulMap[key] = r.upload ?? 0;
+      if (key) ulMap[key] = r.upload ?? 0;
-    }
+    }
-  }
+  }
-  return dlData.map(r => {
+  return dlData.map(r => {
-    const label = r.application_category?.label ?? String(r.application_category ?? 'Unknown');
+    const label = r.application_category?.label ?? String(r.application_category ?? 'Unknown');
-    return {
+    return {
-      category_label : label,
+      category_label : label,
-      download       : r.download ?? 0,
+      download       : r.download ?? 0,
-      upload         : ulMap[label] ?? 0,
+      upload         : ulMap[label] ?? 0,
-      flows          : r.flows ?? 0,
+      flows          : r.flows ?? 0,
-    };
+    };
-  });
+  });
-}
+}
-
+
-async function fetchTlsVersions(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
+async function fetchTlsVersions(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
-  const [dlData, ulData] = await Promise.all([
+  const [dlData, ulData] = await Promise.all([
-    netifyFetch('/data/stats/top/tls_version/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    netifyFetch('/data/stats/top/tls_version/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
-    netifyFetch('/data/stats/top/tls_version/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    netifyFetch('/data/stats/top/tls_version/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
-  ]);
+  ]);
-  if (!dlData) return [];
+  if (!dlData) return [];
-  const ulMap = {};
+  const ulMap = {};
-  if (ulData) {
+  if (ulData) {
-    for (const r of ulData) {
+    for (const r of ulData) {
-      const key = r.tls_version?.label ?? r.tls_version?.code ?? r.tls_version;
+      const key = r.tls_version?.label ?? r.tls_version?.code ?? r.tls_version;
-      if (key) ulMap[key] = r.upload ?? 0;
+      if (key) ulMap[key] = r.upload ?? 0;
-    }
+    }
-  }
+  }
-  return dlData.map(r => {
+  return dlData.map(r => {
-    const label = r.tls_version?.label ?? r.tls_version?.code ?? String(r.tls_version ?? 'Unknown');
+    const label = r.tls_version?.label ?? r.tls_version?.code ?? String(r.tls_version ?? 'Unknown');
-    return {
+    return {
-      tls_version : label,
+      tls_version : label,
-      download    : r.download ?? 0,
+      download    : r.download ?? 0,
-      upload      : ulMap[label] ?? 0,
+      upload      : ulMap[label] ?? 0,
-      flows       : r.flows ?? 0,
+      flows       : r.flows ?? 0,
-    };
+    };
-  });
+  });
-}
+}
-
+
-async function fetchTlsCiphers(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
+async function fetchTlsCiphers(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
-  const [dlData, ulData] = await Promise.all([
+  const [dlData, ulData] = await Promise.all([
-    netifyFetch('/data/stats/top/tls_cipher/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    netifyFetch('/data/stats/top/tls_cipher/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
-    netifyFetch('/data/stats/top/tls_cipher/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    netifyFetch('/data/stats/top/tls_cipher/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
-  ]);
+  ]);
-  if (!dlData) return [];
+  if (!dlData) return [];
-  const ulMap = {};
+  const ulMap = {};
-  if (ulData) {
+  if (ulData) {
-    for (const r of ulData) {
+    for (const r of ulData) {
-      const key = r.tls_cipher?.label ?? r.tls_cipher?.code ?? r.tls_cipher;
+      const key = r.tls_cipher?.label ?? r.tls_cipher?.code ?? r.tls_cipher;
-      if (key) ulMap[key] = r.upload ?? 0;
+      if (key) ulMap[key] = r.upload ?? 0;
-    }
+    }
-  }
+  }
-  return dlData.map(r => {
+  return dlData.map(r => {
-    const label = r.tls_cipher?.label ?? r.tls_cipher?.code ?? String(r.tls_cipher ?? 'Unknown');
+    const label = r.tls_cipher?.label ?? r.tls_cipher?.code ?? String(r.tls_cipher ?? 'Unknown');
-    return {
+    return {
-      tls_cipher : label,
+      tls_cipher : label,
-      download   : r.download ?? 0,
+      download   : r.download ?? 0,
-      upload     : ulMap[label] ?? 0,
+      upload     : ulMap[label] ?? 0,
-      flows      : r.flows ?? 0,
+      flows      : r.flows ?? 0,
-    };
+    };
-  });
+  });
-}
+}
-
+
-async function fetchTlsSecurity(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
+async function fetchTlsSecurity(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
-  const [dlData, ulData] = await Promise.all([
+  const [dlData, ulData] = await Promise.all([
-    netifyFetch('/data/stats/top/tls_security/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    netifyFetch('/data/stats/top/tls_security/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
-    netifyFetch('/data/stats/top/tls_security/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    netifyFetch('/data/stats/top/tls_security/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
-  ]);
+  ]);
-  if (!dlData) return [];
+  if (!dlData) return [];
-  const ulMap = {};
+  const ulMap = {};
-  if (ulData) {
+  if (ulData) {
-    for (const r of ulData) {
+    for (const r of ulData) {
-      const key = r.tls_security?.label ?? r.tls_security?.code ?? r.tls_security;
+      const key = r.tls_security?.label ?? r.tls_security?.code ?? r.tls_security;
-      if (key) ulMap[key] = r.upload ?? 0;
+      if (key) ulMap[key] = r.upload ?? 0;
-    }
+    }
-  }
+  }
-  return dlData.map(r => {
+  return dlData.map(r => {
-    const label = r.tls_security?.label ?? r.tls_security?.code ?? String(r.tls_security ?? 'Unknown');
+    const label = r.tls_security?.label ?? r.tls_security?.code ?? String(r.tls_security ?? 'Unknown');
-    return {
+    return {
-      tls_security : label,
+      tls_security : label,
-      download     : r.download ?? 0,
+      download     : r.download ?? 0,
-      upload       : ulMap[label] ?? 0,
+      upload       : ulMap[label] ?? 0,
-      flows        : r.flows ?? 0,
+      flows        : r.flows ?? 0,
-    };
+    };
-  });
+  });
-}
+}
-
+
-async function fetchTopCountries(interval = 1440, limit = 100, agentUuid = null, siteUuid = null) {
+async function fetchTopCountries(interval = 1440, limit = 100, agentUuid = null, siteUuid = null) {
-  const [dlData, ulData] = await Promise.all([
+  const [dlData, ulData] = await Promise.all([
-    netifyFetch('/data/stats/top/country/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    netifyFetch('/data/stats/top/country/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
-    netifyFetch('/data/stats/top/country/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    netifyFetch('/data/stats/top/country/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
-  ]);
+  ]);
-  if (!dlData) return [];
+  if (!dlData) return [];
-  const ulMap = {};
+  const ulMap = {};
-  if (ulData) {
+  if (ulData) {
-    for (const r of ulData) {
+    for (const r of ulData) {
-      const cc = r.country?.code;
+      const cc = r.country?.code;
-      if (cc) ulMap[cc] = r.upload ?? 0;
+      if (cc) ulMap[cc] = r.upload ?? 0;
-    }
+    }
-  }
+  }
-  return dlData.map(r => {
+  return dlData.map(r => {
-    return {
+    return {
-      country_code: r.country?.code ?? 'Unknown',
+      country_code: r.country?.code ?? 'Unknown',
-      country_name: r.country?.label ?? '',
+      country_name: r.country?.label ?? '',
-      download:     r.download ?? 0,
+      download:     r.download ?? 0,
-      upload:       ulMap[r.country?.code] ?? 0,
+      upload:       ulMap[r.country?.code] ?? 0,
-      flows:        r.flows ?? 0,
+      flows:        r.flows ?? 0,
-    };
+    };
-  }).filter(r => r.country_code);
+  }).filter(r => r.country_code);
-}
+}
-
+
-async function fetchTopProperty(fieldName, interval, limit, agentUuid, siteUuid) {
+async function fetchTopProperty(fieldName, interval, limit, agentUuid, siteUuid) {
-  const [dlData, ulData] = await Promise.all([
+  const [dlData, ulData] = await Promise.all([
-    netifyFetch(`/data/stats/top/${fieldName}/download`, { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    netifyFetch(`/data/stats/top/${fieldName}/download`, { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
-    netifyFetch(`/data/stats/top/${fieldName}/upload`,   { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
+    netifyFetch(`/data/stats/top/${fieldName}/upload`,   { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
-  ]);
+  ]);
-  if (!dlData) return [];
+  if (!dlData) return [];
-  const ulMap = {};
+  const ulMap = {};
-  if (ulData) {
+  if (ulData) {
-    for (const r of ulData) {
+    for (const r of ulData) {
-      const item = r[fieldName];
+      const item = r[fieldName];
-      const key = item?.hash ?? item?.name ?? item?.label ?? String(item ?? '');
+      const key = item?.hash ?? item?.name ?? item?.label ?? String(item ?? '');
-      if (key) ulMap[key] = r.upload ?? 0;
+      if (key) ulMap[key] = r.upload ?? 0;
-    }
+    }
-  }
+  }
-  return dlData.map(r => {
+  return dlData.map(r => {
-    const item = r[fieldName];
+    const item = r[fieldName];
-    const key = item?.hash ?? item?.name ?? item?.label ?? String(item || 'Unknown');
+    const key = item?.hash ?? item?.name ?? item?.label ?? String(item || 'Unknown');
-    const label = item?.label ?? key;
+    const label = item?.label ?? key;
-    return {
+    return {
-      key,
+      key,
-      label,
+      label,
-      download: r.download ?? 0,
+      download: r.download ?? 0,
-      upload:   ulMap[key] ?? ulMap[label] ?? 0,
+      upload:   ulMap[key] ?? ulMap[label] ?? 0,
-      flows:    r.flows ?? 0,
+      flows:    r.flows ?? 0,
-    };
+    };
-  });
+  });
-}
+}
-
+
-function mapProp(data, keyName) {
+function mapProp(data, keyName) {
-  return data.map(d => ({
+  return data.map(d => ({
-    [keyName]: d.key,
+    [keyName]: d.key,
-    download: d.download,
+    download: d.download,
-    upload:   d.upload,
+    upload:   d.upload,
-    flows:    d.flows,
+    flows:    d.flows,
-  }));
+  }));
-}
+}
-
+
-async function fetchDhcpFingerprints(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+async function fetchDhcpFingerprints(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
-  return mapProp(await fetchTopProperty('dhcp_class', interval, limit, agentUuid, siteUuid), 'fingerprint');
+  return mapProp(await fetchTopProperty('dhcp_class', interval, limit, agentUuid, siteUuid), 'fingerprint');
-}
+}
-
+
-async function fetchHttpUserAgents(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+async function fetchHttpUserAgents(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
-  return mapProp(await fetchTopProperty('http_useragent', interval, limit, agentUuid, siteUuid), 'user_agent');
+  return mapProp(await fetchTopProperty('http_useragent', interval, limit, agentUuid, siteUuid), 'user_agent');
-}
+}
-
+
-async function fetchBittorrentHashes(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+async function fetchBittorrentHashes(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
-  const data = await fetchTopProperty('bittorrent_info_hash', interval, limit, agentUuid, siteUuid);
+  const data = await fetchTopProperty('bittorrent_info_hash', interval, limit, agentUuid, siteUuid);
-  return data.map(d => ({
+  return data.map(d => ({
-    info_hash: d.key,
+    info_hash: d.key,
-    label:     d.label,
+    label:     d.label,
-    download:  d.download,
+    download:  d.download,
-    upload:    d.upload,
+    upload:    d.upload,
-    flows:     d.flows,
+    flows:     d.flows,
-  }));
+  }));
-}
+}
-
+
-async function fetchSniHostnames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+async function fetchSniHostnames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
-  return mapProp(await fetchTopProperty('tls_sni', interval, limit, agentUuid, siteUuid), 'sni_hostname');
+  return mapProp(await fetchTopProperty('tls_sni', interval, limit, agentUuid, siteUuid), 'sni_hostname');
-}
+}
-
+
-async function fetchSslServerCn(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+async function fetchSslServerCn(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
-  return mapProp(await fetchTopProperty('ssl_server_cn', interval, limit, agentUuid, siteUuid), 'ssl_server_cn');
+  return mapProp(await fetchTopProperty('ssl_server_cn', interval, limit, agentUuid, siteUuid), 'ssl_server_cn');
-}
+}
-
+
-async function fetchQuicHostnames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+async function fetchQuicHostnames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
-  return mapProp(await fetchTopProperty('quic_hostname', interval, limit, agentUuid, siteUuid), 'quic_hostname');
+  return mapProp(await fetchTopProperty('quic_hostname', interval, limit, agentUuid, siteUuid), 'quic_hostname');
-}
+}
-
+
-async function fetchSshClients(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+async function fetchSshClients(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
-  return mapProp(await fetchTopProperty('ssh_client', interval, limit, agentUuid, siteUuid), 'ssh_client');
+  return mapProp(await fetchTopProperty('ssh_client', interval, limit, agentUuid, siteUuid), 'ssh_client');
-}
+}
-
+
-async function fetchSshServers(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+async function fetchSshServers(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
-  return mapProp(await fetchTopProperty('ssh_server', interval, limit, agentUuid, siteUuid), 'ssh_server');
+  return mapProp(await fetchTopProperty('ssh_server', interval, limit, agentUuid, siteUuid), 'ssh_server');
-}
+}
-
+
-async function fetchMdnsHostnames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+async function fetchMdnsHostnames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
-  return mapProp(await fetchTopProperty('mdns_hostname', interval, limit, agentUuid, siteUuid), 'mdns_hostname');
+  return mapProp(await fetchTopProperty('mdns_hostname', interval, limit, agentUuid, siteUuid), 'mdns_hostname');
-}
+}
-
+
-async function fetchTopProtocols(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+async function fetchTopProtocols(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
-  return mapProp(await fetchTopProperty('ip_protocol', interval, limit, agentUuid, siteUuid), 'protocol_label');
+  return mapProp(await fetchTopProperty('ip_protocol', interval, limit, agentUuid, siteUuid), 'protocol_label');
-}
+}
-
+
-async function fetchSslSubjectAltNames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
+async function fetchSslSubjectAltNames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
-  return mapProp(await fetchTopProperty('ssl_subject_alt_name', interval, limit, agentUuid, siteUuid), 'alt_name');
+  return mapProp(await fetchTopProperty('ssl_subject_alt_name', interval, limit, agentUuid, siteUuid), 'alt_name');
-}
+}
-
+
-module.exports = {
+module.exports = {
-  fetchTopAppCategories,
+  fetchTopAppCategories,
-  fetchTlsVersions,
+  fetchTlsVersions,
-  fetchTlsCiphers,
+  fetchTlsCiphers,
-  fetchTlsSecurity,
+  fetchTlsSecurity,
-  fetchTopCountries,
+  fetchTopCountries,
-  fetchDhcpFingerprints,
+  fetchDhcpFingerprints,
-  fetchHttpUserAgents,
+  fetchHttpUserAgents,
-  fetchBittorrentHashes,
+  fetchBittorrentHashes,
-  fetchSniHostnames,
+  fetchSniHostnames,
-  fetchSslServerCn,
+  fetchSslServerCn,
-  fetchQuicHostnames,
+  fetchQuicHostnames,
-  fetchSshClients,
+  fetchSshClients,
-  fetchSshServers,
+  fetchSshServers,
-  fetchMdnsHostnames,
+  fetchMdnsHostnames,
-  fetchTopProtocols,
+  fetchTopProtocols,
-  fetchSslSubjectAltNames,
+  fetchSslSubjectAltNames,
-};
+};

proxy/package.json

@@ -1,19 +1,19 @@
-{
+{
-  "name": "backone-proxy",
+  "name": "backone-proxy",
-  "version": "1.0.0",
+  "version": "1.0.0",
-  "description": "BackOne DPI Proxy Server - Fetches from DPI API, filters per agent_uuid, stores to MongoDB",
+  "description": "BackOne DPI Proxy Server - Fetches from DPI API, filters per agent_uuid, stores to MongoDB",
-  "main": "index.js",
+  "main": "index.js",
-  "scripts": {
+  "scripts": {
-    "start": "node index.js",
+    "start": "node index.js",
-    "start:bun": "bun run index.js",
+    "start:bun": "bun run index.js",
-    "dev": "nodemon index.js"
+    "dev": "nodemon index.js"
-  },
+  },
-  "dependencies": {
+  "dependencies": {
-    "axios": "^1.6.2",
+    "axios": "^1.6.2",
-    "cors": "^2.8.5",
+    "cors": "^2.8.5",
-    "dotenv": "^16.3.1",
+    "dotenv": "^16.3.1",
-    "express": "^4.18.2",
+    "express": "^4.18.2",
-    "mongoose": "^8.0.3",
+    "mongoose": "^8.0.3",
-    "node-cron": "^3.0.3"
+    "node-cron": "^3.0.3"
-  }
+  }
-}
+}

proxy/scheduler.js

@@ -1,124 +1,129 @@
-// proxy/scheduler.js
+// proxy/scheduler.js
-// Cron scheduler for automatic data collection from DPI API
+// Cron scheduler for automatic data collection from DPI API
-// Runs every 5 minutes, collecting data for all agents or a specific agent.
+// Runs every 5 minutes, collecting data for all agents or a specific agent.
-
+
-const cron = require('node-cron');
+const cron = require('node-cron');
-const { collectAllAgents, collectSpecificAgent, collectSpecificAgents } = require('./collector');
+const { collectAllAgents, collectSpecificAgent, collectSpecificAgents } = require('./collector');
-
+const { syncApplicationDictionary } = require('./netifyClient');
-// Mode: 'all' = collect all agents, 'agent' = collect one specific agent, 'agents' = collect multiple agents
+
-const COLLECT_MODE    = process.env.PROXY_COLLECT_MODE || 'all';
+// Mode: 'all' = collect all agents, 'agent' = collect one specific agent, 'agents' = collect multiple agents
-const SPECIFIC_AGENT  = process.env.PROXY_AGENT_UUID   || null;
+const COLLECT_MODE    = process.env.PROXY_COLLECT_MODE || 'all';
-const SPECIFIC_AGENTS = (process.env.PROXY_AGENT_UUIDS || '').split(',').map(s => s.trim()).filter(Boolean);
+const SPECIFIC_AGENT  = process.env.PROXY_AGENT_UUID   || null;
-const AGENT_DELAY_MS  = parseInt(process.env.PROXY_AGENT_DELAY_MS || '5000');
+const SPECIFIC_AGENTS = (process.env.PROXY_AGENT_UUIDS || '').split(',').map(s => s.trim()).filter(Boolean);
-const CRON_SCHEDULE   = process.env.PROXY_CRON_SCHEDULE || '*/5 * * * *';
+const AGENT_DELAY_MS  = parseInt(process.env.PROXY_AGENT_DELAY_MS || '5000');
-
+const CRON_SCHEDULE   = process.env.PROXY_CRON_SCHEDULE || '*/5 * * * *';
-// Capacity logging is an expensive full-scan aggregation. Run it at most once per
+
-// interval (default 24h) instead of every collection cycle to reduce CPU/DB load.
+// Capacity logging is an expensive full-scan aggregation. Run it at most once per
-const CAPACITY_LOG_INTERVAL_MS = parseInt(process.env.PROXY_CAPACITY_LOG_INTERVAL_MS || String(24 * 60 * 60 * 1000));
+// interval (default 24h) instead of every collection cycle to reduce CPU/DB load.
-
+const CAPACITY_LOG_INTERVAL_MS = parseInt(process.env.PROXY_CAPACITY_LOG_INTERVAL_MS || String(24 * 60 * 60 * 1000));
-let isRunning = false;
+
-let lastRunAt = null;
+let isRunning = false;
-let lastRunResult = null;
+let lastRunAt = null;
-let runCount = 0;
+let lastRunResult = null;
-let lastCapacityLogAt = 0;
+let runCount = 0;
-
+let lastCapacityLogAt = 0;
-/**
+
- * Execute one collection cycle (called by cron and manual trigger).
+/**
- * Prevents concurrent runs with isRunning guard.
+ * Execute one collection cycle (called by cron and manual trigger).
- */
+ * Prevents concurrent runs with isRunning guard.
-async function runCollection() {
+ */
-  if (isRunning) {
+async function runCollection() {
-    console.log('[Scheduler] Skipping - previous run still in progress');
+  if (isRunning) {
-    return { skipped: true, reason: 'already_running' };
+    console.log('[Scheduler] Skipping - previous run still in progress');
-  }
+    return { skipped: true, reason: 'already_running' };
-
+  }
-  isRunning = true;
+
-  lastRunAt = new Date();
+  isRunning = true;
-  runCount++;
+  lastRunAt = new Date();
-
+  runCount++;
-  try {
+
-    let result;
+  try {
-    if (COLLECT_MODE === 'agent' && SPECIFIC_AGENT) {
+    let result;
-      console.log(`[Scheduler] Run #${runCount} - Mode: SPECIFIC AGENT (${SPECIFIC_AGENT})`);
+    if (COLLECT_MODE === 'agent' && SPECIFIC_AGENT) {
-      result = await collectSpecificAgent(SPECIFIC_AGENT);
+      console.log(`[Scheduler] Run #${runCount} - Mode: SPECIFIC AGENT (${SPECIFIC_AGENT})`);
-    } else if (COLLECT_MODE === 'agents' && SPECIFIC_AGENTS.length > 0) {
+      result = await collectSpecificAgent(SPECIFIC_AGENT);
-      console.log(`[Scheduler] Run #${runCount} - Mode: SPECIFIC AGENTS (${SPECIFIC_AGENTS.join(', ')})`);
+    } else if (COLLECT_MODE === 'agents' && SPECIFIC_AGENTS.length > 0) {
-      result = await collectSpecificAgents(SPECIFIC_AGENTS, AGENT_DELAY_MS);
+      console.log(`[Scheduler] Run #${runCount} - Mode: SPECIFIC AGENTS (${SPECIFIC_AGENTS.join(', ')})`);
-    } else {
+      result = await collectSpecificAgents(SPECIFIC_AGENTS, AGENT_DELAY_MS);
-      console.log(`[Scheduler] Run #${runCount} - Mode: ALL AGENTS`);
+    } else {
-      result = await collectAllAgents();
+      console.log(`[Scheduler] Run #${runCount} - Mode: ALL AGENTS`);
-    }
+      result = await collectAllAgents();
-    lastRunResult = { ...result, run_count: runCount };
+    }
-    
+    lastRunResult = { ...result, run_count: runCount };
-    // Log MongoDB database capacity usage (expensive full-scan aggregation).
+    
-    // Only run periodically (default: every 24h) to avoid high CPU/DB load each cycle.
+    // Log MongoDB database capacity usage (expensive full-scan aggregation).
-    const now = Date.now();
+    // Only run periodically (default: every 24h) to avoid high CPU/DB load each cycle.
-    if (now - lastCapacityLogAt >= CAPACITY_LOG_INTERVAL_MS) {
+    const now = Date.now();
-      lastCapacityLogAt = now;
+    if (now - lastCapacityLogAt >= CAPACITY_LOG_INTERVAL_MS) {
-      const { logCapacityStats } = require('./db/capacityTracker');
+      lastCapacityLogAt = now;
-      await logCapacityStats(`[PROXY] [MongoDB] Capacity Used after Run #${runCount}:`);
+      const { logCapacityStats } = require('./db/capacityTracker');
-    }
+      await logCapacityStats(`[PROXY] [MongoDB] Capacity Used after Run #${runCount}:`);
-    
+    }
-    return lastRunResult;
+    
-  } catch (err) {
+    return lastRunResult;
-    console.error('[Scheduler] Unhandled error during collection:', err.message);
+  } catch (err) {
-    lastRunResult = { success: false, error: err.message, run_count: runCount };
+    console.error('[Scheduler] Unhandled error during collection:', err.message);
-    return lastRunResult;
+    lastRunResult = { success: false, error: err.message, run_count: runCount };
-  } finally {
+    return lastRunResult;
-    isRunning = false;
+  } finally {
-  }
+    isRunning = false;
-}
+  }
-
+}
-/**
+
- * Start the scheduler (cron job + immediate first run).
+/**
- */
+ * Start the scheduler (cron job + immediate first run).
-function startScheduler() {
+ */
-  console.log(`[Scheduler] Starting proxy data collector`);
+function startScheduler() {
-  const modeLabel = COLLECT_MODE === 'agent'
+  console.log(`[Scheduler] Starting proxy data collector`);
-    ? `SPECIFIC AGENT (${SPECIFIC_AGENT})`
+  const modeLabel = COLLECT_MODE === 'agent'
-    : COLLECT_MODE === 'agents'
+    ? `SPECIFIC AGENT (${SPECIFIC_AGENT})`
-    ? `SPECIFIC AGENTS (${SPECIFIC_AGENTS.join(', ')})`
+    : COLLECT_MODE === 'agents'
-    : 'ALL AGENTS';
+    ? `SPECIFIC AGENTS (${SPECIFIC_AGENTS.join(', ')})`
-  console.log(`[Scheduler] Mode     : ${modeLabel}`);
+    : 'ALL AGENTS';
-  console.log(`[Scheduler] Schedule : ${CRON_SCHEDULE} (every 5 minutes by default)`);
+  console.log(`[Scheduler] Mode     : ${modeLabel}`);
-
+  console.log(`[Scheduler] Schedule : ${CRON_SCHEDULE} (every 5 minutes by default)`);
-  // Validate cron expression
+
-  if (!cron.validate(CRON_SCHEDULE)) {
+  // Validate cron expression
-    console.error(`[Scheduler] Invalid cron expression: "${CRON_SCHEDULE}". Using default.`);
+  if (!cron.validate(CRON_SCHEDULE)) {
-  }
+    console.error(`[Scheduler] Invalid cron expression: "${CRON_SCHEDULE}". Using default.`);
-
+  }
-  // Start recurring cron job
+
-  cron.schedule(CRON_SCHEDULE, () => {
+  // Start recurring cron job
-    runCollection().catch(err => console.error('[Scheduler] Cron error:', err.message));
+  cron.schedule(CRON_SCHEDULE, () => {
-  });
+    runCollection().catch(err => console.error('[Scheduler] Cron error:', err.message));
-
+  });
-  console.log('[Scheduler] Cron job registered. Starting initial collection...');
+
-  
+  console.log('[Scheduler] Cron job registered. Starting initial collection...');
-  // Initial run immediately on startup (async, do not block server start)
+  
-  setTimeout(async () => {
+  // Initial run immediately on startup (async, do not block server start)
-    // 1. Sync dictionary first
+  setTimeout(async () => {
-    // await netifyClient.syncApplicationDictionary();
+    // 1. Sync dictionary first
-    
+    try {
-    // 2. Start normal telemetry collection
+      await syncApplicationDictionary();
-    runCollection().catch(err => console.error('[Scheduler] Initial run error:', err.message));
+    } catch (err) {
-  }, 2000);
+      console.error('[Scheduler] Error syncing application dictionary:', err.message);
-}
+    }
-
+    
-/**
+    // 2. Start normal telemetry collection
- * Get current scheduler status (for REST API endpoint).
+    runCollection().catch(err => console.error('[Scheduler] Initial run error:', err.message));
- */
+  }, 2000);
-function getStatus() {
+}
-  return {
+
-    is_running:     isRunning,
+/**
-    run_count:      runCount,
+ * Get current scheduler status (for REST API endpoint).
-    last_run_at:    lastRunAt?.toISOString() ?? null,
+ */
-    collect_mode:   COLLECT_MODE,
+function getStatus() {
-    agent_uuid:     SPECIFIC_AGENT,
+  return {
-    agent_uuids:    COLLECT_MODE === 'agents' ? SPECIFIC_AGENTS : [],
+    is_running:     isRunning,
-    agent_delay_ms: AGENT_DELAY_MS,
+    run_count:      runCount,
-    cron_schedule:  CRON_SCHEDULE,
+    last_run_at:    lastRunAt?.toISOString() ?? null,
-    last_result:    lastRunResult,
+    collect_mode:   COLLECT_MODE,
-  };
+    agent_uuid:     SPECIFIC_AGENT,
-}
+    agent_uuids:    COLLECT_MODE === 'agents' ? SPECIFIC_AGENTS : [],
-
+    agent_delay_ms: AGENT_DELAY_MS,
-module.exports = { startScheduler, runCollection, getStatus };
+    cron_schedule:  CRON_SCHEDULE,
+    last_result:    lastRunResult,
+  };
+}
+
+module.exports = { startScheduler, runCollection, getStatus };

proxy/test_api.js

@@ -0,0 +1,71 @@
+// test_api.js - Tests the actual metadata-detail API endpoint
+// Run: node proxy/test_api.js
+const http = require('http');
+
+function request(path) {
+  return new Promise((resolve, reject) => {
+    const options = {
+      hostname: 'localhost',
+      port: 3001,
+      path,
+      method: 'GET',
+    };
+    const req = http.request(options, res => {
+      let body = '';
+      res.on('data', chunk => body += chunk);
+      res.on('end', () => {
+        try { resolve({ status: res.statusCode, data: JSON.parse(body) }); }
+        catch (e) { resolve({ status: res.statusCode, raw: body }); }
+      });
+    });
+    req.on('error', reject);
+    req.end();
+  });
+}
+
+async function main() {
+  // Test 1: netbios_hostname for 10.6.10.44
+  console.log('\n=== TEST 1: netbios_hostname=10.6.10.44 ===');
+  try {
+    const r1 = await request('/api/dashboard/metadata-detail?type=netbios_hostname&value=10.6.10.44');
+    console.log('Status:', r1.status);
+    if (r1.data) {
+      console.log('Count:', r1.data.count);
+      console.log('First 3 rows:', JSON.stringify(r1.data.data?.slice(0, 3), null, 2));
+    } else {
+      console.log('Raw:', r1.raw?.slice(0, 500));
+    }
+  } catch (e) {
+    console.log('ERROR (maybe backend is on different port):', e.message);
+  }
+
+  // Test 2: Try port 3000 (Next.js API routes)
+  console.log('\n=== TEST 2: via Next.js port 3000 ===');
+  try {
+    const r2 = await request('/api/dashboard/metadata-detail?type=netbios_hostname&value=10.6.10.44');
+    const options2 = { hostname: 'localhost', port: 3000, path: '/api/dashboard/metadata-detail?type=netbios_hostname&value=10.6.10.44', method: 'GET' };
+    const r3 = await new Promise((resolve, reject) => {
+      const req = http.request(options2, res => {
+        let body = '';
+        res.on('data', chunk => body += chunk);
+        res.on('end', () => {
+          try { resolve({ status: res.statusCode, data: JSON.parse(body) }); }
+          catch (e) { resolve({ status: res.statusCode, raw: body?.slice(0, 500) }); }
+        });
+      });
+      req.on('error', reject);
+      req.end();
+    });
+    console.log('Port 3000 - Status:', r3.status);
+    if (r3.data) {
+      console.log('Count:', r3.data.count);
+      console.log('First 3 rows:', JSON.stringify(r3.data.data?.slice(0, 3), null, 2));
+    } else {
+      console.log('Raw:', r3.raw);
+    }
+  } catch (e) {
+    console.log('Port 3000 ERROR:', e.message);
+  }
+}
+
+main().catch(console.error);

proxy/test_db.js

@@ -0,0 +1 @@
+const mongoose = require('mongoose'); require('dotenv').config({ path: '../.env.local' }); const { LookupApp } = require('./models/Schemas'); async function test() { await mongoose.connect(process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017/backone_dpi'); const sample = await LookupApp.findOne({ tag: /youtube/i }).lean(); console.log(JSON.stringify(sample, null, 2)); await mongoose.disconnect(); } test().catch(console.error);

proxy/test_metadata_detail.js

@@ -0,0 +1,89 @@
+// test_metadata_detail.js - Test after restart
+// Run: node proxy/test_metadata_detail.js
+const http = require('http');
+
+function post(path, body) {
+  return new Promise((resolve, reject) => {
+    const data = JSON.stringify(body);
+    const options = {
+      hostname: 'localhost', port: 3001, path, method: 'POST',
+      headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data) },
+    };
+    const req = http.request(options, res => {
+      let buf = '';
+      res.on('data', c => buf += c);
+      res.on('end', () => {
+        try { resolve({ status: res.statusCode, headers: res.headers, data: JSON.parse(buf) }); }
+        catch { resolve({ status: res.statusCode, raw: buf }); }
+      });
+    });
+    req.on('error', reject);
+    req.write(data);
+    req.end();
+  });
+}
+
+function get(path, cookie) {
+  return new Promise((resolve, reject) => {
+    const options = {
+      hostname: 'localhost', port: 3001, path, method: 'GET',
+      headers: cookie ? { Cookie: cookie } : {},
+    };
+    const req = http.request(options, res => {
+      let buf = '';
+      res.on('data', c => buf += c);
+      res.on('end', () => {
+        try { resolve({ status: res.statusCode, data: JSON.parse(buf) }); }
+        catch { resolve({ status: res.statusCode, raw: buf?.slice(0, 300) }); }
+      });
+    });
+    req.on('error', reject);
+    req.end();
+  });
+}
+
+async function main() {
+  // Step 1: Login to get cookie
+  console.log('=== Step 1: Login ===');
+  const login = await post('/api/auth/login', { username: 'admin', password: 'admin123' });
+  console.log('Login status:', login.status);
+  
+  const setCookie = login.headers?.['set-cookie'];
+  let cookie = '';
+  if (setCookie) {
+    cookie = setCookie.map(c => c.split(';')[0]).join('; ');
+    console.log('Cookie obtained:', cookie.slice(0, 60) + '...');
+  } else {
+    console.log('No cookie received. Auth response:', JSON.stringify(login.data));
+    // Try with a known admin credential
+  }
+
+  // Step 2: Test health
+  console.log('\n=== Step 2: Health Check ===');
+  const health = await get('/api/health', cookie);
+  console.log('Health:', health.status, JSON.stringify(health.data));
+
+  // Step 3: Test metadata-detail netbios_hostname
+  console.log('\n=== Step 3: metadata-detail netbios_hostname=10.6.10.44 ===');
+  const r = await get('/api/dashboard/metadata-detail?type=netbios_hostname&value=10.6.10.44', cookie);
+  console.log('Status:', r.status);
+  if (r.data) {
+    console.log('Count (should be 1):', r.data.count);
+    console.log('Data:', JSON.stringify(r.data.data, null, 2));
+  } else {
+    console.log('Raw:', r.raw);
+  }
+
+  // Step 4: Verify DB has 1 doc for 10.6.10.44
+  console.log('\n=== Step 4: Direct DB verification ===');
+  const mongoose = require('mongoose');
+  const path = require('path');
+  require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
+  await mongoose.connect(process.env.MONGODB_URI || 'mongodb://localhost:27017/backone');
+  const db = mongoose.connection.db;
+  const cnt = await db.collection('devicestats').countDocuments({ ip_address: '10.6.10.44' });
+  console.log('DB count for 10.6.10.44:', cnt, '(expected: 1)');
+  await mongoose.disconnect();
+}
+
+main().catch(console.error);

proxy/test_sync_proxy.js

@@ -0,0 +1 @@
+const mongoose = require('mongoose'); require('dotenv').config({ path: '../.env.local' }); const { syncApplicationDictionary } = require('./netifyClient'); const { LookupApp } = require('./models/Schemas'); async function test() { await mongoose.connect(process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017/backone_dpi'); console.log('Connected to DB'); await syncApplicationDictionary(); const count = await LookupApp.countDocuments(); console.log('Total LookupApps in DB:', count); await mongoose.disconnect(); } test().catch(console.error);

proxy/verify_fix.js

@@ -0,0 +1,56 @@
+// verify_fix.js - Directly verify the MongoDB aggregation returns correct results
+// This simulates what the backend metadata-detail endpoint does after the fix.
+// Run: node proxy/verify_fix.js
+const path = require('path');
+require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
+const mongoose = require('mongoose');
+
+async function run() {
+  await mongoose.connect(process.env.MONGODB_URI || 'mongodb://localhost:27017/backone');
+  const db = mongoose.connection.db;
+  const col = db.collection('devicestats');
+
+  const testValues = ['10.6.10.44'];
+  // Also find other common device_labels to test
+  const sample = await col.find({}).limit(20).toArray();
+  const labels = [...new Set(sample.map(d => d.device_label).filter(Boolean))];
+  console.log('Sample device_labels to test:', labels.slice(0, 5));
+
+  for (const value of [...testValues, ...labels.slice(0, 3)]) {
+    // Count raw docs matching
+    const rawCount = await col.countDocuments({ device_label: value });
+    
+    // Simulate the new aggregation pipeline
+    const aggResult = await col.aggregate([
+      { $match: { device_label: value } },
+      { $sort: { timestamp: -1 } },
+      { $group: {
+          _id: '$ip_address',
+          mac_address:  { $first: '$mac_address' },
+          device_label: { $first: '$device_label' },
+          download:     { $max:   '$download' },
+          upload:       { $max:   '$upload' },
+      }},
+      { $sort: { download: -1 } },
+    ]).toArray();
+
+    const status = rawCount > aggResult.length ? '✅ FIXED (was duplicated)' : '✓ OK';
+    console.log(`\ndevice_label="${value}": raw=${rawCount} rows → aggregated=${aggResult.length} unique devices ${status}`);
+    if (aggResult.length > 0) {
+      console.log('  First result:', JSON.stringify(aggResult[0], null, 2));
+    }
+  }
+
+  // Verify unique index exists
+  const indexes = await col.indexes();
+  const uniqueIdx = indexes.find(i => i.unique && i.key.agent_uuid && i.key.ip_address);
+  console.log('\n=== Unique Index on (agent_uuid, ip_address):', uniqueIdx ? `✅ EXISTS (${uniqueIdx.name})` : '❌ MISSING');
+
+  // Final count
+  const total = await col.countDocuments();
+  console.log('=== Total DeviceStat docs:', total);
+
+  await mongoose.disconnect();
+}
+
+run().catch(err => { console.error(err.message); process.exit(1); });

public/backone-logo.svg

@@ -1,21 +1,21 @@
-<svg width="200" height="200" viewBox="0 0 200 200" xmlns="http://www.w3.org/2000/svg">
+<svg width="200" height="200" viewBox="0 0 200 200" xmlns="http://www.w3.org/2000/svg">
-  <defs>
+  <defs>
-    <linearGradient id="bgGradient" x1="0%" y1="0%" x2="100%" y2="100%">
+    <linearGradient id="bgGradient" x1="0%" y1="0%" x2="100%" y2="100%">
-      <stop offset="0%" stop-color="#f00a38" />
+      <stop offset="0%" stop-color="#f00a38" />
-      <stop offset="50%" stop-color="#7a0799" />
+      <stop offset="50%" stop-color="#7a0799" />
-      <stop offset="100%" stop-color="#0400ff" />
+      <stop offset="100%" stop-color="#0400ff" />
-    </linearGradient>
+    </linearGradient>
-  </defs>
+  </defs>
-  
+  
-  <!-- Gradient Circle Background -->
+  <!-- Gradient Circle Background -->
-  <circle cx="100" cy="100" r="100" fill="url(#bgGradient)" />
+  <circle cx="100" cy="100" r="100" fill="url(#bgGradient)" />
-  
+  
-  <!-- White Abstract Shape (BackOne Logo) -->
+  <!-- White Abstract Shape (BackOne Logo) -->
-  <path d="M 50 45
+  <path d="M 50 45
-           L 90 45
+           L 90 45
-           C 145 45, 165 65, 165 105
+           C 145 45, 165 65, 165 105
-           C 165 155, 110 180, 55 180
+           C 165 155, 110 180, 55 180
-           C 115 160, 125 100, 50 95
+           C 115 160, 125 100, 50 95
-           Z" 
+           Z" 
-        fill="#FFFFFF" />
+        fill="#FFFFFF" />
-</svg>
+</svg>