// proxy/dataRetention.js // ───────────────────────────────────────────────────────────────────────────── // Pruning process for BackOne MongoDB data retention. // Removes telemetry records older than 30 days to conserve database space. // Also removes device/flow records that are outside the agent's configured subnet. // ───────────────────────────────────────────────────────────────────────────── const mongoose = require('mongoose'); const Schemas = require('./models/Schemas'); /** * Prune all time-series documents older than 30 days. */ async function pruneOldData() { const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000); const timeString = thirtyDaysAgo.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB'; console.log(`[Collector] [Retention] Checking for telemetry data older than 30 days (before ${timeString})...`); // Prune from all collections in Schemas except CustomDeviceLabel const collections = Object.keys(Schemas).filter(name => name !== 'CustomDeviceLabel'); for (const name of collections) { try { const Model = Schemas[name]; if (typeof Model.deleteMany === 'function') { const res = await Model.deleteMany({ timestamp: { $lt: thirtyDaysAgo } }); if (res.deletedCount > 0) { console.log(`[Collector] [Retention] ✓ Cleaned up ${res.deletedCount} old records from ${name}`); } } } catch (err) { console.error(`[Collector] [Retention] ✗ Failed to prune ${name}: ${err.message}`); } } // Also prune out-of-subnet data await pruneOutOfSubnetData(); } // ───────────────────────────────────────────────────────────────────────────── // Subnet-based cleanup: remove devices/flows that are outside the agent's // configured allowed_subnets. Runs after every collection cycle automatically. // ───────────────────────────────────────────────────────────────────────────── function _ipToLong(ip) { return ip.split('.').reduce((acc, o) => (acc << 8) + parseInt(o, 10), 0) >>> 0; } function _ipMatchesSubnets(ip, subnets) { if (!ip || ip.includes(':')) return false; // skip IPv6 if (!subnets || subnets.length === 0) return true; // no restriction return subnets.some(s => { if (s.includes('/')) { try { const [r, b] = s.split('/'); const bits = parseInt(b, 10); if (isNaN(bits) || bits < 0 || bits > 32) return false; const mask = bits === 0 ? 0 : (~0 << (32 - bits)) >>> 0; return (_ipToLong(ip) & mask) === (_ipToLong(r) & mask); } catch { return false; } } return ip.startsWith(s + '.') || ip === s; }); } async function pruneOutOfSubnetData() { try { const db = mongoose.connection.db; const agents = await db.collection('agent_registry') .find({ allowed_subnets: { $exists: true, $not: { $size: 0 } } }) .toArray(); for (const agent of agents) { const uuid = agent.uuid; const subnets = (agent.allowed_subnets || []).map(s => s.trim()).filter(Boolean); if (subnets.length === 0) continue; // devicestats const devIps = await db.collection('devicestats').distinct('ip_address', { agent_uuid: uuid }); const badDevIps = devIps.filter(ip => !_ipMatchesSubnets(ip, subnets)); if (badDevIps.length > 0) { const r1 = await db.collection('devicestats').deleteMany({ agent_uuid: uuid, ip_address: { $in: badDevIps } }); const r2 = await db.collection('deviceappstats').deleteMany({ agent_uuid: uuid, ip_address: { $in: badDevIps } }); if (r1.deletedCount + r2.deletedCount > 0) { console.log(`[Collector] [Subnet] ${uuid}: removed ${r1.deletedCount} device + ${r2.deletedCount} deviceapp records (${badDevIps.length} bad IPs)`); } } // flows const flowIps = await db.collection('flows').distinct('src_ip', { agent_uuid: uuid }); const badFlowIps = flowIps.filter(ip => !_ipMatchesSubnets(ip, subnets)); if (badFlowIps.length > 0) { const r = await db.collection('flows').deleteMany({ agent_uuid: uuid, src_ip: { $in: badFlowIps } }); if (r.deletedCount > 0) { console.log(`[Collector] [Subnet] ${uuid}: removed ${r.deletedCount} flow records (${badFlowIps.length} bad IPs)`); } } } } catch (err) { console.error('[Collector] [Subnet] pruneOutOfSubnetData error:', err.message); } } module.exports = { pruneOldData, pruneOutOfSubnetData };