From 52282e3822b9fc01d2f6ead08684cdfbd3e1ed69 Mon Sep 17 00:00:00 2001 From: dsutanto Date: Tue, 14 Jul 2026 15:08:50 +0700 Subject: [PATCH 1/5] Make it proxy as independent docker image --- proxy/db/capacityTracker.js | 59 +++++++++++++++++++++++++++++++++++++ proxy/index.js | 8 ++--- proxy/scheduler.js | 2 +- 3 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 proxy/db/capacityTracker.js diff --git a/proxy/db/capacityTracker.js b/proxy/db/capacityTracker.js new file mode 100644 index 0000000..7f385a2 --- /dev/null +++ b/proxy/db/capacityTracker.js @@ -0,0 +1,59 @@ +// proxy/db/capacityTracker.js +// ───────────────────────────────────────────────────────────────────────────── +// MongoDB Capacity & Data Size Breakdown per Network Agent. +// Measures logical document sizes per agent_uuid across all collections. +// ───────────────────────────────────────────────────────────────────────────── + +const mongoose = require('mongoose'); + +async function logCapacityStats(prefix = '[MongoDB]') { + try { + if (!mongoose.connection || !mongoose.connection.db) { + return; + } + const db = mongoose.connection.db; + + const stats = await db.command({ dbStats: 1 }); + const dataSizeMB = (stats.dataSize / (1024 * 1024)).toFixed(2); + const storageSizeMB = (stats.storageSize / (1024 * 1024)).toFixed(2); + console.log(`${prefix} Capacity Used: Data Size = ${dataSizeMB} MB, Storage Size = ${storageSizeMB} MB`); + + const agentSizes = {}; + const collections = await db.listCollections().toArray(); + + for (const colInfo of collections) { + const colName = colInfo.name; + if (colName.startsWith('system.')) continue; + const col = db.collection(colName); + + const sampleDoc = await col.findOne({ agent_uuid: { $ne: null } }); + if (!sampleDoc) continue; + + const pipeline = [ + { $project: { agent_uuid: 1, docSize: { $bsonSize: "$$ROOT" } } }, + { $group: { _id: "$agent_uuid", totalBytes: { $sum: "$docSize" } } } + ]; + + const results = await col.aggregate(pipeline).toArray(); + for (const res of results) { + const agent = res._id || 'Unknown'; + agentSizes[agent] = (agentSizes[agent] || 0) + res.totalBytes; + } + } + + const sortedAgents = Object.entries(agentSizes) + .map(([agent, bytes]) => ({ agent, sizeMB: parseFloat((bytes / (1024 * 1024)).toFixed(2)) })) + .sort((a, b) => b.sizeMB - a.sizeMB); + + if (sortedAgents.length > 0) { + console.log(`${prefix} Data Size Breakdown per Agent:`); + for (const { agent, sizeMB } of sortedAgents) { + console.log(` - ${agent}: ${sizeMB.toFixed(2)} MB`); + } + } + } catch (err) { + console.warn(`${prefix} Could not retrieve DB capacity breakdown:`, err.message); + } +} + +module.exports = { logCapacityStats }; diff --git a/proxy/index.js b/proxy/index.js index a143e29..13efa27 100644 --- a/proxy/index.js +++ b/proxy/index.js @@ -43,7 +43,7 @@ async function connectDB() { socketTimeoutMS: 30000, }); console.log('[MongoDB] ✓ Connected successfully to', MONGODB_URI); - const { logCapacityStats } = require('../backend/db/capacityTracker'); + const { logCapacityStats } = require('./db/capacityTracker'); await logCapacityStats('[MongoDB]'); return true; } catch (err) { @@ -165,8 +165,8 @@ app.get('/agents', async (req, res) => { if (mongoose.connection.readyState !== 1) { return res.status(503).json({ ok: false, error: 'MongoDB not connected.' }); } - const { DashboardSummary } = require('../backend/models/Schemas'); - const agents = await DashboardSummary.distinct('agent_uuid'); + const { Summary } = require('./models/Schemas'); + const agents = await Summary.distinct('agent_uuid'); res.json({ ok: true, data: agents }); } catch (err) { res.status(500).json({ ok: false, error: err.message }); @@ -183,7 +183,7 @@ app.get('/domain-details', async (req, res) => { if (!domain) return res.status(400).json({ ok: false, error: 'domain is required' }); try { - const { Flow, DeviceStat } = require('../backend/models/Schemas'); + const { Flow, DeviceStat } = require('./models/Schemas'); const filter = { domain: domain }; if (agentUuid) { diff --git a/proxy/scheduler.js b/proxy/scheduler.js index dc6da44..e1f14b1 100644 --- a/proxy/scheduler.js +++ b/proxy/scheduler.js @@ -55,7 +55,7 @@ async function runCollection() { const now = Date.now(); if (now - lastCapacityLogAt >= CAPACITY_LOG_INTERVAL_MS) { lastCapacityLogAt = now; - const { logCapacityStats } = require('../backend/db/capacityTracker'); + const { logCapacityStats } = require('./db/capacityTracker'); await logCapacityStats(`[PROXY] [MongoDB] Capacity Used after Run #${runCount}:`); } From ff54929a54276381227b0a8955d0809f21a7b86f Mon Sep 17 00:00:00 2001 From: dsutanto Date: Tue, 14 Jul 2026 15:14:16 +0700 Subject: [PATCH 2/5] INFO.md for proxy --- proxy/INFO.md | 108 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 proxy/INFO.md diff --git a/proxy/INFO.md b/proxy/INFO.md new file mode 100644 index 0000000..e3ab5db --- /dev/null +++ b/proxy/INFO.md @@ -0,0 +1,108 @@ +# BackOne DPI Proxy — Deployment Reference + +Standalone Docker image for collecting Netify DPI data and writing to MongoDB. + +--- + +## Environment Variables + +### Required + +| Variable | Description | +|---|---| +| `NETIFY_SITE_UUIDS` | Comma-separated Netify site UUIDs (or single `NETIFY_SITE_UUID`) | +| `NETIFY_TOKEN` | Netify JWT token (or `NETIFY_JWT_TOKEN` / `NETIFY_API_KEY`) | + +### MongoDB + +| Variable | Default | Description | +|---|---|---| +| `MONGODB_URI` | `mongodb://127.0.0.1:27017/backone_dpi` | MongoDB connection string | + +### Collection Mode + +| Variable | Default | Description | +|---|---|---| +| `PROXY_COLLECT_MODE` | `all` | `all` = all agents, `agent` = single agent, `agents` = list of agents | +| `PROXY_AGENT_UUID` | _(none)_ | Single agent UUID (required if `mode=agent`) | +| `PROXY_AGENT_UUIDS` | _(none)_ | Comma-separated agent UUIDs (required if `mode=agents`) | +| `PROXY_AGENT_DELAY_MS` | `5000` | Delay (ms) between each agent collection to avoid rate-limiting | + +### Scheduling + +| Variable | Default | Description | +|---|---|---| +| `PROXY_CRON_SCHEDULE` | `*/5 * * * *` | Cron expression for collection interval | +| `PROXY_CAPACITY_LOG_INTERVAL_MS` | `86400000` | How often to log DB capacity usage (default: 24h) | + +### Limits + +| Variable | Default | Description | +|---|---|---| +| `PROXY_FLOW_LIMIT` | `1000000` | Max flows to fetch per agent per cycle | +| `PROXY_PORT` | `4000` | REST API listen port | + +### Netify API + +| Variable | Default | Description | +|---|---|---| +| `NETIFY_INFORMATICS_BASE_URL` | `https://informatics.netify.ai/api/v1` | Netify API base URL | + +--- + +## Docker Run Example + +```bash +docker run -d \ + --name backone_proxy \ + -p 4000:4000 \ + -e MONGODB_URI=mongodb://host.docker.internal:27017/backone_dpi \ + -e NETIFY_SITE_UUIDS=site-uuid-1,site-uuid-2 \ + -e NETIFY_TOKEN=your-jwt-token \ + -e PROXY_COLLECT_MODE=agents \ + -e PROXY_AGENT_UUIDS=agent-uuid-1,agent-uuid-2,agent-uuid-3 \ + backone-proxy +``` + +## Docker Compose Example + +```yaml +services: + proxy: + build: ./proxy + container_name: backone_proxy + restart: always + ports: + - "4000:4000" + environment: + - MONGODB_URI=mongodb://mongodb:27017/backone_dpi + - PROXY_PORT=4000 + - PROXY_COLLECT_MODE=all + - PROXY_COLLECT_MODE=${PROXY_COLLECT_MODE:-all} + - PROXY_AGENT_UUID=${PROXY_AGENT_UUID:-} + - PROXY_AGENT_UUIDS=${PROXY_AGENT_UUIDS:-} + - PROXY_AGENT_DELAY_MS=${PROXY_AGENT_DELAY_MS:-5000} + - PROXY_CRON_SCHEDULE=${PROXY_CRON_SCHEDULE:-*/5 * * * *} + - NETIFY_SITE_UUIDS=${NETIFY_SITE_UUIDS} + - NETIFY_TOKEN=${NETIFY_TOKEN} + - NETIFY_INFORMATICS_BASE_URL=${NETIFY_INFORMATICS_BASE_URL:-https://informatics.netify.ai/api/v1} +``` + +## REST API Endpoints + +| Method | Endpoint | Description | +|---|---|---| +| `GET` | `/health` | Liveness check (MongoDB status) | +| `GET` | `/status` | Scheduler status, mode, last run | +| `GET` | `/agents` | List agent UUIDs in MongoDB | +| `POST` | `/collect/all` | Manual trigger — all agents | +| `POST` | `/collect/:uuid` | Manual trigger — single agent | +| `POST` | `/collect/agents` | Manual trigger — multiple agents `{"uuids":[...], "delay_ms":5000}` | +| `GET` | `/latest` | Latest data from all collections (debug) | +| `GET` | `/domain-details?domain=...` | IP/MAC details for a domain | + +## Health Check + +```bash +curl http://localhost:4000/health +``` From c0a4d0c7e22afff2e29d5d4bdd641d5e8e0b1776 Mon Sep 17 00:00:00 2001 From: dsutanto Date: Tue, 14 Jul 2026 15:15:26 +0700 Subject: [PATCH 3/5] Add bun support --- proxy/Dockerfile.bun | 20 ++++++++++++++++++++ proxy/package.json | 1 + 2 files changed, 21 insertions(+) create mode 100644 proxy/Dockerfile.bun diff --git a/proxy/Dockerfile.bun b/proxy/Dockerfile.bun new file mode 100644 index 0000000..80a56ad --- /dev/null +++ b/proxy/Dockerfile.bun @@ -0,0 +1,20 @@ +FROM oven/bun:1-alpine + +WORKDIR /app + +# Install dependencies first (layer caching) +# bun install is compatible with npm package.json / package-lock.json +COPY package*.json ./ +RUN bun install --production + +# Copy application source +COPY . . + +# Expose proxy REST API port +EXPOSE 4000 + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \ + CMD bun -e "require('http').get('http://localhost:4000/health', r => r.statusCode === 200 ? process.exit(0) : process.exit(1)).on('error', () => process.exit(1))" + +CMD ["bun", "run", "index.js"] diff --git a/proxy/package.json b/proxy/package.json index 285d3fb..99f1e35 100644 --- a/proxy/package.json +++ b/proxy/package.json @@ -5,6 +5,7 @@ "main": "index.js", "scripts": { "start": "node index.js", + "start:bun": "bun run index.js", "dev": "nodemon index.js" }, "dependencies": { From a839fd7798388066c413c144854845f3c24e6eb2 Mon Sep 17 00:00:00 2001 From: dsutanto Date: Tue, 14 Jul 2026 19:11:41 +0700 Subject: [PATCH 4/5] Fix bug in proxy --- proxy/netifyClient.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/proxy/netifyClient.js b/proxy/netifyClient.js index a8ed62a..149b608 100644 --- a/proxy/netifyClient.js +++ b/proxy/netifyClient.js @@ -134,8 +134,9 @@ async function fetchFlows(limit = 10000, agentUuid = null, siteUuid = null) { const sniList = []; if (sniRaw && Array.isArray(sniRaw)) { for (const r of sniRaw) { - const sni = r.tls_sni; - if (sni && sni.trim() !== '') sniList.push(sni.replace(/^\*\./, '').trim()); + const sni = r.tls_sni ?? r.label; + const sniStr = (sni === null || sni === undefined) ? '' : String(sni); + if (sniStr.trim() !== '') sniList.push(sniStr.replace(/^\*\./, '').trim()); } } return raw.map(r => { From 9e7e8cc6d9913c1d1bc990f650e96d03f353a4b6 Mon Sep 17 00:00:00 2001 From: dsutanto Date: Tue, 14 Jul 2026 19:20:05 +0700 Subject: [PATCH 5/5] Fix Proxy 2 --- proxy/netifyClient.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/proxy/netifyClient.js b/proxy/netifyClient.js index 149b608..e17fd31 100644 --- a/proxy/netifyClient.js +++ b/proxy/netifyClient.js @@ -134,9 +134,8 @@ async function fetchFlows(limit = 10000, agentUuid = null, siteUuid = null) { const sniList = []; if (sniRaw && Array.isArray(sniRaw)) { for (const r of sniRaw) { - const sni = r.tls_sni ?? r.label; - const sniStr = (sni === null || sni === undefined) ? '' : String(sni); - if (sniStr.trim() !== '') sniList.push(sniStr.replace(/^\*\./, '').trim()); + const sni = typeof r.tls_sni === 'object' ? r.tls_sni?.label : r.tls_sni; + if (sni && typeof sni === 'string' && sni.trim() !== '') sniList.push(sni.replace(/^\*\./, '').trim()); } } return raw.map(r => {