diff --git a/package.json b/package.json index f666f37..5fb1a70 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "lint": "eslint", "backend": "node backend/server.js", "proxy": "node proxy/index.js", + "proxy:bun": "bun proxy/index.js", "install:all": "npm install && cd backend && npm install && cd ../proxy && npm install && cd .." }, "dependencies": { diff --git a/proxy/collectorHelperDpi2.js b/proxy/collectorHelperDpi2.js index 66422f0..035e328 100644 --- a/proxy/collectorHelperDpi2.js +++ b/proxy/collectorHelperDpi2.js @@ -81,7 +81,8 @@ async function collectDevicesAndApps(agentUuid, timestamp, SITE_UUID, netify, la } async function collectFlows(agentUuid, timestamp, SITE_UUID, netify, label, ipToMacMap) { - const flows = await netify.fetchFlows(10000, agentUuid); + const flowLimit = parseInt(process.env.PROXY_FLOW_LIMIT || '10000'); + const flows = await netify.fetchFlows(flowLimit, agentUuid); if (flows && flows.length > 0) { const flowDocs = flows.map(f => { const mac = f.src_mac || ipToMacMap[f.src_ip] || generateMacFromIp(f.src_ip); @@ -136,9 +137,15 @@ async function collectEvents(agentUuid, timestamp, SITE_UUID, netify, label) { const existingSet = new Set(existing); const macToAgentMap = {}; - const storedDevices = await DeviceStat.find({ site_uuid: SITE_UUID }).lean(); - for (const d of storedDevices) { - if (d.mac_address && d.agent_uuid) macToAgentMap[d.mac_address] = d.agent_uuid; + const eventMacs = [...new Set(events.map(e => e.mac_address).filter(Boolean))]; + if (eventMacs.length > 0) { + const storedDevices = await DeviceStat.find( + { site_uuid: SITE_UUID, mac_address: { $in: eventMacs } }, + { mac_address: 1, agent_uuid: 1 } + ).lean(); + for (const d of storedDevices) { + if (d.mac_address && d.agent_uuid) macToAgentMap[d.mac_address] = d.agent_uuid; + } } const eventDocs = events.filter(e => e.event_id === null || !existingSet.has(e.event_id)).map(e => { diff --git a/proxy/scheduler.js b/proxy/scheduler.js index 4fb33f7..5cc7d2d 100644 --- a/proxy/scheduler.js +++ b/proxy/scheduler.js @@ -10,10 +10,15 @@ const COLLECT_MODE = process.env.PROXY_COLLECT_MODE || 'all'; const SPECIFIC_AGENT = process.env.PROXY_AGENT_UUID || null; const CRON_SCHEDULE = process.env.PROXY_CRON_SCHEDULE || '*/5 * * * *'; +// Capacity logging is an expensive full-scan aggregation. Run it at most once per +// interval (default 24h) instead of every collection cycle to reduce CPU/DB load. +const CAPACITY_LOG_INTERVAL_MS = parseInt(process.env.PROXY_CAPACITY_LOG_INTERVAL_MS || String(24 * 60 * 60 * 1000)); + let isRunning = false; let lastRunAt = null; let lastRunResult = null; let runCount = 0; +let lastCapacityLogAt = 0; /** * Execute one collection cycle (called by cron and manual trigger). @@ -39,11 +44,16 @@ async function runCollection() { result = await collectAllAgents(); } lastRunResult = { ...result, run_count: runCount }; - - // Log MongoDB database capacity usage after collection run - const { logCapacityStats } = require('../backend/db/capacityTracker'); - await logCapacityStats(`[PROXY] [MongoDB] Capacity Used after Run #${runCount}:`); - + + // Log MongoDB database capacity usage (expensive full-scan aggregation). + // Only run periodically (default: every 24h) to avoid high CPU/DB load each cycle. + const now = Date.now(); + if (now - lastCapacityLogAt >= CAPACITY_LOG_INTERVAL_MS) { + lastCapacityLogAt = now; + const { logCapacityStats } = require('../backend/db/capacityTracker'); + await logCapacityStats(`[PROXY] [MongoDB] Capacity Used after Run #${runCount}:`); + } + return lastRunResult; } catch (err) { console.error('[Scheduler] Unhandled error during collection:', err.message);