105 lines
5.0 KiB
JavaScript
105 lines
5.0 KiB
JavaScript
// proxy/dataRetention.js
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Pruning process for BackOne MongoDB data retention.
|
|
// Removes telemetry records older than 30 days to conserve database space.
|
|
// Also removes device/flow records that are outside the agent's configured subnet.
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
const mongoose = require('mongoose');
|
|
const Schemas = require('./models/Schemas');
|
|
|
|
/**
|
|
* Prune all time-series documents older than 30 days.
|
|
*/
|
|
async function pruneOldData() {
|
|
const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
|
|
const timeString = thirtyDaysAgo.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
|
|
console.log(`[Collector] [Retention] Checking for telemetry data older than 30 days (before ${timeString})...`);
|
|
|
|
// Prune from all collections in Schemas except CustomDeviceLabel
|
|
const collections = Object.keys(Schemas).filter(name => name !== 'CustomDeviceLabel');
|
|
|
|
for (const name of collections) {
|
|
try {
|
|
const Model = Schemas[name];
|
|
if (typeof Model.deleteMany === 'function') {
|
|
const res = await Model.deleteMany({ timestamp: { $lt: thirtyDaysAgo } });
|
|
if (res.deletedCount > 0) {
|
|
console.log(`[Collector] [Retention] ✓ Cleaned up ${res.deletedCount} old records from ${name}`);
|
|
}
|
|
}
|
|
} catch (err) {
|
|
console.error(`[Collector] [Retention] ✗ Failed to prune ${name}: ${err.message}`);
|
|
}
|
|
}
|
|
|
|
// Also prune out-of-subnet data
|
|
await pruneOutOfSubnetData();
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Subnet-based cleanup: remove devices/flows that are outside the agent's
|
|
// configured allowed_subnets. Runs after every collection cycle automatically.
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
function _ipToLong(ip) {
|
|
return ip.split('.').reduce((acc, o) => (acc << 8) + parseInt(o, 10), 0) >>> 0;
|
|
}
|
|
|
|
function _ipMatchesSubnets(ip, subnets) {
|
|
if (!ip || ip.includes(':')) return false; // skip IPv6
|
|
if (!subnets || subnets.length === 0) return true; // no restriction
|
|
return subnets.some(s => {
|
|
if (s.includes('/')) {
|
|
try {
|
|
const [r, b] = s.split('/');
|
|
const bits = parseInt(b, 10);
|
|
if (isNaN(bits) || bits < 0 || bits > 32) return false;
|
|
const mask = bits === 0 ? 0 : (~0 << (32 - bits)) >>> 0;
|
|
return (_ipToLong(ip) & mask) === (_ipToLong(r) & mask);
|
|
} catch { return false; }
|
|
}
|
|
return ip.startsWith(s + '.') || ip === s;
|
|
});
|
|
}
|
|
|
|
async function pruneOutOfSubnetData() {
|
|
try {
|
|
const db = mongoose.connection.db;
|
|
const agents = await db.collection('agent_registry')
|
|
.find({ allowed_subnets: { $exists: true, $not: { $size: 0 } } })
|
|
.toArray();
|
|
|
|
for (const agent of agents) {
|
|
const uuid = agent.uuid;
|
|
const subnets = (agent.allowed_subnets || []).map(s => s.trim()).filter(Boolean);
|
|
if (subnets.length === 0) continue;
|
|
|
|
// devicestats
|
|
const devIps = await db.collection('devicestats').distinct('ip_address', { agent_uuid: uuid });
|
|
const badDevIps = devIps.filter(ip => !_ipMatchesSubnets(ip, subnets));
|
|
if (badDevIps.length > 0) {
|
|
const r1 = await db.collection('devicestats').deleteMany({ agent_uuid: uuid, ip_address: { $in: badDevIps } });
|
|
const r2 = await db.collection('deviceappstats').deleteMany({ agent_uuid: uuid, ip_address: { $in: badDevIps } });
|
|
if (r1.deletedCount + r2.deletedCount > 0) {
|
|
console.log(`[Collector] [Subnet] ${uuid}: removed ${r1.deletedCount} device + ${r2.deletedCount} deviceapp records (${badDevIps.length} bad IPs)`);
|
|
}
|
|
}
|
|
|
|
// flows
|
|
const flowIps = await db.collection('flows').distinct('src_ip', { agent_uuid: uuid });
|
|
const badFlowIps = flowIps.filter(ip => !_ipMatchesSubnets(ip, subnets));
|
|
if (badFlowIps.length > 0) {
|
|
const r = await db.collection('flows').deleteMany({ agent_uuid: uuid, src_ip: { $in: badFlowIps } });
|
|
if (r.deletedCount > 0) {
|
|
console.log(`[Collector] [Subnet] ${uuid}: removed ${r.deletedCount} flow records (${badFlowIps.length} bad IPs)`);
|
|
}
|
|
}
|
|
}
|
|
} catch (err) {
|
|
console.error('[Collector] [Subnet] pruneOutOfSubnetData error:', err.message);
|
|
}
|
|
}
|
|
|
|
module.exports = { pruneOldData, pruneOutOfSubnetData };
|