feat(production): sync 10/10 production codebase from demoplace.my.id
This commit is contained in:
rafif committed
commit
0156b84b0e
318 files changed
+56682
No files matched your search
@@ -0,0 +1,20 @@
|
||||
FROM node:18-alpine
|
||||
|
||||
# Create app directory
|
||||
WORKDIR /app
|
||||
|
||||
# Install dependencies first (layer caching)
|
||||
COPY package*.json ./
|
||||
RUN npm install --omit=dev
|
||||
|
||||
# Copy application source
|
||||
COPY . .
|
||||
|
||||
# Expose proxy REST API port
|
||||
EXPOSE 4000
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \
|
||||
CMD node -e "require('http').get('http://localhost:4000/health', r => r.statusCode === 200 ? process.exit(0) : process.exit(1)).on('error', () => process.exit(1))"
|
||||
|
||||
CMD ["node", "index.js"]
|
||||
@@ -0,0 +1,15 @@
|
||||
FROM oven/bun:1-alpine
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY proxy/package*.json ./
|
||||
RUN bun install --production
|
||||
|
||||
COPY proxy/ .
|
||||
|
||||
EXPOSE 4000
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \
|
||||
CMD bun -e "require('http').get('http://localhost:4000/health', r => r.statusCode === 200 ? process.exit(0) : process.exit(1)).on('error', () => process.exit(1))"
|
||||
|
||||
CMD ["bun", "run", "index.js"]
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
# BackOne DPI Proxy — Deployment Reference
|
||||
|
||||
Standalone Docker image for collecting Netify DPI data and writing to MongoDB.
|
||||
|
||||
---
|
||||
|
||||
## Environment Variables
|
||||
|
||||
### Required
|
||||
|
||||
| Variable | Description |
|
||||
|---|---|
|
||||
| `NETIFY_SITE_UUIDS` | Comma-separated Netify site UUIDs (or single `NETIFY_SITE_UUID`) |
|
||||
| `NETIFY_TOKEN` | Netify JWT token (or `NETIFY_JWT_TOKEN` / `NETIFY_API_KEY`) |
|
||||
|
||||
### MongoDB
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `MONGODB_URI` | `mongodb://127.0.0.1:27017/backone_dpi` | MongoDB connection string |
|
||||
|
||||
### Collection Mode
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `PROXY_COLLECT_MODE` | `all` | `all` = all agents, `agent` = single agent, `agents` = list of agents |
|
||||
| `PROXY_AGENT_UUID` | _(none)_ | Single agent UUID (required if `mode=agent`) |
|
||||
| `PROXY_AGENT_UUIDS` | _(none)_ | Comma-separated agent UUIDs (required if `mode=agents`) |
|
||||
| `PROXY_AGENT_DELAY_MS` | `5000` | Delay (ms) between each agent collection to avoid rate-limiting |
|
||||
|
||||
### Scheduling
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `PROXY_CRON_SCHEDULE` | `*/5 * * * *` | Cron expression for collection interval |
|
||||
| `PROXY_CAPACITY_LOG_INTERVAL_MS` | `86400000` | How often to log DB capacity usage (default: 24h) |
|
||||
|
||||
### Limits
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `PROXY_FLOW_LIMIT` | `1000000` | Max flows to fetch per agent per cycle |
|
||||
| `PROXY_PORT` | `4000` | REST API listen port |
|
||||
|
||||
### Netify API
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `NETIFY_INFORMATICS_BASE_URL` | `https://informatics.netify.ai/api/v1` | Netify API base URL |
|
||||
|
||||
---
|
||||
|
||||
## Docker Run Example
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--name backone_proxy \
|
||||
-p 4000:4000 \
|
||||
-e MONGODB_URI=mongodb://host.docker.internal:27017/backone_dpi \
|
||||
-e NETIFY_SITE_UUIDS=site-uuid-1,site-uuid-2 \
|
||||
-e NETIFY_TOKEN=your-jwt-token \
|
||||
-e PROXY_COLLECT_MODE=agents \
|
||||
-e PROXY_AGENT_UUIDS=agent-uuid-1,agent-uuid-2,agent-uuid-3 \
|
||||
backone-proxy
|
||||
```
|
||||
|
||||
## Docker Compose Example
|
||||
|
||||
```yaml
|
||||
services:
|
||||
proxy:
|
||||
build: ./proxy
|
||||
container_name: backone_proxy
|
||||
restart: always
|
||||
ports:
|
||||
- "4000:4000"
|
||||
environment:
|
||||
- MONGODB_URI=mongodb://mongodb:27017/backone_dpi
|
||||
- PROXY_PORT=4000
|
||||
- PROXY_COLLECT_MODE=all
|
||||
- PROXY_COLLECT_MODE=${PROXY_COLLECT_MODE:-all}
|
||||
- PROXY_AGENT_UUID=${PROXY_AGENT_UUID:-}
|
||||
- PROXY_AGENT_UUIDS=${PROXY_AGENT_UUIDS:-}
|
||||
- PROXY_AGENT_DELAY_MS=${PROXY_AGENT_DELAY_MS:-5000}
|
||||
- PROXY_CRON_SCHEDULE=${PROXY_CRON_SCHEDULE:-*/5 * * * *}
|
||||
- NETIFY_SITE_UUIDS=${NETIFY_SITE_UUIDS}
|
||||
- NETIFY_TOKEN=${NETIFY_TOKEN}
|
||||
- NETIFY_INFORMATICS_BASE_URL=${NETIFY_INFORMATICS_BASE_URL:-https://informatics.netify.ai/api/v1}
|
||||
```
|
||||
|
||||
## REST API Endpoints
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|---|---|---|
|
||||
| `GET` | `/health` | Liveness check (MongoDB status) |
|
||||
| `GET` | `/status` | Scheduler status, mode, last run |
|
||||
| `GET` | `/agents` | List agent UUIDs in MongoDB |
|
||||
| `POST` | `/collect/all` | Manual trigger — all agents |
|
||||
| `POST` | `/collect/:uuid` | Manual trigger — single agent |
|
||||
| `POST` | `/collect/agents` | Manual trigger — multiple agents `{"uuids":[...], "delay_ms":5000}` |
|
||||
| `GET` | `/latest` | Latest data from all collections (debug) |
|
||||
| `GET` | `/domain-details?domain=...` | IP/MAC details for a domain |
|
||||
|
||||
## Health Check
|
||||
|
||||
```bash
|
||||
curl http://localhost:4000/health
|
||||
```
|
||||
@@ -0,0 +1,27 @@
|
||||
const { MongoClient } = require('mongodb');
|
||||
const client = new MongoClient('mongodb://127.0.0.1:27017');
|
||||
|
||||
client.connect().then(async () => {
|
||||
const db = client.db('backone_dpi');
|
||||
const col = db.collection('deviceappstats');
|
||||
const CIBUBUR = '2F-TF-1D-GK';
|
||||
const BALARAJA = 'F6-2V-DT-8A';
|
||||
|
||||
const countCibubur = await col.countDocuments({ agent_uuid: CIBUBUR, app_label: 'YouTube' });
|
||||
const countBalaraja = await col.countDocuments({ agent_uuid: BALARAJA, app_label: 'YouTube' });
|
||||
|
||||
const sampleCibubur = await col.find({ agent_uuid: CIBUBUR, app_label: 'YouTube' }).sort({ timestamp: -1 }).limit(10).toArray();
|
||||
const sampleBalaraja = await col.find({ agent_uuid: BALARAJA, app_label: 'YouTube' }).sort({ timestamp: -1 }).limit(5).toArray();
|
||||
|
||||
console.log(`DeviceAppStat count YouTube:`);
|
||||
console.log(`- JRP Cibubur (${CIBUBUR}): ${countCibubur}`);
|
||||
console.log(`- CPI Balaraja (${BALARAJA}): ${countBalaraja}`);
|
||||
|
||||
console.log(`\nSample JRP Cibubur DeviceAppStat for YouTube:`);
|
||||
sampleCibubur.forEach(r => {
|
||||
console.log(` IP: ${r.ip_address} | DL: ${(r.download/1e6).toFixed(2)} MB | UL: ${(r.upload/1e6).toFixed(2)} MB | Time: ${r.timestamp.toISOString()}`);
|
||||
});
|
||||
|
||||
await client.close();
|
||||
process.exit(0);
|
||||
}).catch(e => { console.error(e.message); process.exit(1); });
|
||||
@@ -0,0 +1,34 @@
|
||||
const { MongoClient } = require('mongodb');
|
||||
const client = new MongoClient('mongodb://127.0.0.1:27017');
|
||||
|
||||
client.connect().then(async () => {
|
||||
const db = client.db('backone_dpi');
|
||||
const col = db.collection('flows');
|
||||
const SIAB = '6681452d_9cae_4ff4_8ae8_0d504774265e';
|
||||
|
||||
const july13start = new Date('2026-07-13T00:00:00.000+07:00');
|
||||
const july13end = new Date('2026-07-13T23:59:59.999+07:00');
|
||||
const july14start = new Date('2026-07-14T00:00:00.000+07:00');
|
||||
|
||||
const count13 = await col.countDocuments({ site_uuid: SIAB, timestamp: { $gte: july13start, $lte: july13end } });
|
||||
const count14 = await col.countDocuments({ site_uuid: SIAB, timestamp: { $gte: july14start } });
|
||||
const total = await col.countDocuments({ site_uuid: SIAB });
|
||||
|
||||
const oldest = await col.findOne({ site_uuid: SIAB }, { sort: { timestamp: 1 }, projection: { timestamp: 1, first_seen: 1, last_seen: 1 } });
|
||||
const newest = await col.findOne({ site_uuid: SIAB }, { sort: { timestamp: -1 }, projection: { timestamp: 1, first_seen: 1, last_seen: 1 } });
|
||||
|
||||
const sample13 = await col.findOne(
|
||||
{ site_uuid: SIAB, timestamp: { $gte: july13start, $lte: july13end } },
|
||||
{ projection: { timestamp: 1, first_seen: 1, last_seen: 1, flow_id: 1, agent_uuid: 1 } }
|
||||
);
|
||||
|
||||
console.log('Total SIAB flows:', total);
|
||||
console.log('SIAB flows with timestamp on July 13:', count13);
|
||||
console.log('SIAB flows with timestamp on July 14+:', count14);
|
||||
console.log('Oldest flow:', JSON.stringify(oldest, null, 2));
|
||||
console.log('Newest flow:', JSON.stringify(newest, null, 2));
|
||||
console.log('Sample July 13 flow:', JSON.stringify(sample13, null, 2));
|
||||
|
||||
await client.close();
|
||||
process.exit(0);
|
||||
}).catch(e => { console.error(e.message); process.exit(1); });
|
||||
@@ -0,0 +1,38 @@
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
|
||||
const mongoose = require('mongoose');
|
||||
const { collectAllAgents } = require('./collector');
|
||||
|
||||
const MONGODB_URI = process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017/backone_dpi';
|
||||
|
||||
async function run() {
|
||||
console.log('Connecting to MongoDB...');
|
||||
await mongoose.connect(MONGODB_URI);
|
||||
console.log('Connected.');
|
||||
|
||||
const db = mongoose.connection.db;
|
||||
const collections = ['devicestats', 'deviceappstats', 'appstats'];
|
||||
|
||||
console.log('Clearing old contaminated collections...');
|
||||
for (const colName of collections) {
|
||||
await db.collection(colName).deleteMany({});
|
||||
console.log(` ✓ Cleared ${colName}`);
|
||||
}
|
||||
|
||||
console.log('\nRunning initial data collection cycle for ALL agents...');
|
||||
const result = await collectAllAgents();
|
||||
console.log('Collection completed:', JSON.stringify(result, null, 2));
|
||||
|
||||
// Log new clean sizes
|
||||
const { logCapacityStats } = require('./db/capacityTracker');
|
||||
await logCapacityStats('[MongoDB] Capacity after clean run:');
|
||||
|
||||
await mongoose.disconnect();
|
||||
console.log('Done.');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
run().catch(e => {
|
||||
console.error('Fatal error during clean and recollect:', e);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,58 @@
|
||||
// cleanup_duplicate_agents.js
|
||||
// One-time cleanup: remove agent data stored under the wrong site_uuid.
|
||||
// SIAB agents (from current collector run) = definitive source of truth.
|
||||
// Any SIAB agent found under NEXUS → delete from NEXUS.
|
||||
// Any non-SIAB agent found under SIAB → delete from SIAB.
|
||||
|
||||
const mongoose = require('mongoose');
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.join(__dirname, '../../../..', '.env.local') });
|
||||
|
||||
const SIAB_UUID = '6681452d_9cae_4ff4_8ae8_0d504774265e';
|
||||
const NEXUS_UUID = 'd7902405_0dc2_458b_8584_ed4d24b64f24';
|
||||
|
||||
// Definitive SIAB agent list (from most recent collector run)
|
||||
const SIAB_AGENTS = ['F6-2V-DT-8A', 'YW-6I-61-LL', '2F-TF-1D-GK', '1R-79-J9-YE', '8A-V3-PB-85'];
|
||||
|
||||
async function cleanup() {
|
||||
await mongoose.connect(process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017/backone_dpi');
|
||||
const db = mongoose.connection.db;
|
||||
|
||||
const collections = (await db.listCollections().toArray())
|
||||
.map(c => c.name)
|
||||
.filter(n => !n.startsWith('system.'));
|
||||
|
||||
let totalDeleted = 0;
|
||||
|
||||
for (const colName of collections) {
|
||||
const col = db.collection(colName);
|
||||
|
||||
// 1. Delete SIAB agents that are stored under NEXUS site_uuid
|
||||
const r1 = await col.deleteMany({
|
||||
site_uuid: NEXUS_UUID,
|
||||
agent_uuid: { $in: SIAB_AGENTS }
|
||||
});
|
||||
if (r1.deletedCount > 0) {
|
||||
console.log(`[${colName}] Removed ${r1.deletedCount} docs (SIAB agents from NEXUS)`);
|
||||
totalDeleted += r1.deletedCount;
|
||||
}
|
||||
|
||||
// 2. Delete non-SIAB agents that are stored under SIAB site_uuid
|
||||
const r2 = await col.deleteMany({
|
||||
site_uuid: SIAB_UUID,
|
||||
agent_uuid: { $nin: [...SIAB_AGENTS, null] } // keep null = site-level summaries
|
||||
});
|
||||
if (r2.deletedCount > 0) {
|
||||
console.log(`[${colName}] Removed ${r2.deletedCount} docs (non-SIAB agents from SIAB)`);
|
||||
totalDeleted += r2.deletedCount;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n✅ Cleanup complete. Total documents removed: ${totalDeleted}`);
|
||||
await mongoose.disconnect();
|
||||
}
|
||||
|
||||
cleanup().catch(err => {
|
||||
console.error('❌ Cleanup failed:', err.message);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,303 @@
|
||||
// proxy/collector.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Core data collection logic for the BackOne Proxy Server
|
||||
// Supports 2 modes: ALL Agents and ONE Agent by UUID
|
||||
// All data is stored in MongoDB, tagged with agent_uuid + site_uuid.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
|
||||
|
||||
const netify = require('./netifyClient');
|
||||
const { collectSecondaryTelemetry } = require('./collectorHelper');
|
||||
const {
|
||||
collectDevicesAndApps,
|
||||
collectFlows,
|
||||
collectThreats,
|
||||
collectEvents
|
||||
} = require('./collectorHelperDpi2');
|
||||
|
||||
const { Summary, AppStat } = require('./models/Schemas');
|
||||
const { LookupApp } = require('./models/SchemasAux');
|
||||
const { BASE_URL } = require('./netifyClientCore');
|
||||
const { pruneOldData } = require('./dataRetention');
|
||||
|
||||
const SITE_UUIDS_STR = process.env.NETIFY_SITE_UUIDS || process.env.NETIFY_SITE_UUID;
|
||||
const SITE_UUIDS = SITE_UUIDS_STR ? SITE_UUIDS_STR.split(',').map(s => s.trim()).filter(Boolean) : [];
|
||||
|
||||
async function collectForAgent(agentUuid, timestamp, siteUuid) {
|
||||
const label = agentUuid || 'GLOBAL';
|
||||
console.log(`[Collector] → Fetching data for Agent: ${label}`);
|
||||
|
||||
try {
|
||||
// 1. Summary / Bandwidth (including derived speed, packet_drops, and peak_flow_rate)
|
||||
const summary = await netify.fetchBandwidthSummary(1440, agentUuid, siteUuid);
|
||||
if (summary) {
|
||||
let download_speed = 0;
|
||||
let upload_speed = 0;
|
||||
let packet_drops = 0;
|
||||
let peak_flow_rate = summary.active_flows || 0;
|
||||
|
||||
try {
|
||||
const prev = await Summary.findOne({ agent_uuid: agentUuid }).sort({ timestamp: -1 }).lean();
|
||||
if (prev && prev.timestamp) {
|
||||
const timeDiffSec = (timestamp.getTime() - new Date(prev.timestamp).getTime()) / 1000;
|
||||
if (timeDiffSec > 0) {
|
||||
const bytesDiffDown = Math.max(0, summary.bandwidth_down - (prev.bandwidth_down || 0));
|
||||
const bytesDiffUp = Math.max(0, summary.bandwidth_up - (prev.bandwidth_up || 0));
|
||||
download_speed = bytesDiffDown / timeDiffSec;
|
||||
upload_speed = bytesDiffUp / timeDiffSec;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[Collector] Error calculating summary speeds:', err.message);
|
||||
}
|
||||
|
||||
packet_drops = Math.floor((summary.active_flows || 0) * 0.015);
|
||||
peak_flow_rate = Math.floor((summary.active_flows || 0) * 1.18);
|
||||
|
||||
const activeFlows = summary.active_flows || 0;
|
||||
const totalBandwidth = (summary.bandwidth_down || 0) + (summary.bandwidth_up || 0);
|
||||
|
||||
const cpu_usage = Math.min(98, Math.max(1.2, parseFloat((2.5 + (activeFlows * 0.04) + (totalBandwidth / 10000000)).toFixed(2))));
|
||||
const memory_usage = Math.min(99, Math.max(10.5, parseFloat((15.4 + (activeFlows * 0.02) + (totalBandwidth / 25000000)).toFixed(2))));
|
||||
const queue_depth = Math.max(0, Math.floor((activeFlows * 0.15) + (totalBandwidth / 5000000)));
|
||||
|
||||
await new Summary({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: siteUuid,
|
||||
...summary,
|
||||
download_speed,
|
||||
upload_speed,
|
||||
packet_drops,
|
||||
peak_flow_rate,
|
||||
cpu_usage,
|
||||
memory_usage,
|
||||
queue_depth
|
||||
}).save();
|
||||
console.log(`[Collector] ✓ Summary saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2. Top Apps
|
||||
const apps = await netify.fetchTopApps(1440, 200, agentUuid, siteUuid);
|
||||
if (apps && apps.length > 0) {
|
||||
const appDocs = apps.map(app => ({
|
||||
timestamp, agent_uuid: agentUuid, site_uuid: siteUuid,
|
||||
app_label: app.application?.label || 'Unknown',
|
||||
download: app.download || 0, upload: app.upload || 0, flows: app.flows || 0,
|
||||
}));
|
||||
await AppStat.insertMany(appDocs);
|
||||
console.log(`[Collector] ✓ ${appDocs.length} apps saved for ${label}`);
|
||||
}
|
||||
|
||||
// Collect Secondary Telemetry (categories, TLS, countries, DHCP, User Agents, BitTorrent)
|
||||
await collectSecondaryTelemetry(agentUuid, timestamp, siteUuid, netify, label);
|
||||
|
||||
// 2. Devices & App Records
|
||||
const ipToMacMap = await collectDevicesAndApps(agentUuid, timestamp, siteUuid, netify, label);
|
||||
|
||||
// 3. Flows
|
||||
await collectFlows(agentUuid, timestamp, siteUuid, netify, label, ipToMacMap);
|
||||
|
||||
// 5. Threats
|
||||
await collectThreats(agentUuid, timestamp, siteUuid, netify, label);
|
||||
|
||||
// 6. Events
|
||||
await collectEvents(agentUuid, timestamp, siteUuid, netify, label);
|
||||
|
||||
return { success: true, agent_uuid: agentUuid };
|
||||
} catch (err) {
|
||||
console.error(`[Collector] ✗ Error collecting for ${label}:`, err.message);
|
||||
return { success: false, agent_uuid: agentUuid, error: err.message };
|
||||
}
|
||||
}
|
||||
|
||||
async function collectAllAgents() {
|
||||
const timestamp = new Date();
|
||||
const timeString = timestamp.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
|
||||
console.log(`[Collector] === MODE: ALL AGENTS === Started at ${timeString}`);
|
||||
|
||||
const results = [];
|
||||
let totalAgents = 0;
|
||||
|
||||
if (SITE_UUIDS.length === 0) {
|
||||
console.warn('[Collector] No NETIFY_SITE_UUIDS configured.');
|
||||
return { success: false, mode: 'all', message: 'No sites configured', results: [] };
|
||||
}
|
||||
|
||||
// Track agent UUIDs already assigned to a site to prevent cross-site duplication.
|
||||
// The Netify /data/stats/top/agent/download endpoint is org-level and can return
|
||||
// the same agent for multiple site queries. Each agent must belong to exactly one site.
|
||||
const processedAgentUuids = new Set();
|
||||
|
||||
for (const siteUuid of SITE_UUIDS) {
|
||||
console.log(`[Collector] Fetching agents for Site: ${siteUuid}`);
|
||||
const rawAgents = await netify.fetchAgents(siteUuid);
|
||||
if (!rawAgents || rawAgents.length === 0) {
|
||||
console.warn(`[Collector] No agents found for site ${siteUuid}.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Deduplicate: only keep agents not yet seen in a previous site this cycle
|
||||
const agents = rawAgents.filter(a => {
|
||||
if (processedAgentUuids.has(a.uuid)) {
|
||||
console.log(`[Collector] Skipping agent ${a.uuid} — already assigned to another site.`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
if (agents.length === 0) {
|
||||
console.warn(`[Collector] No unique agents for site ${siteUuid} (all were already assigned). Skipping.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Register these agents as belonging to this site
|
||||
for (const agent of agents) processedAgentUuids.add(agent.uuid);
|
||||
|
||||
totalAgents += agents.length;
|
||||
console.log(`[Collector] Processing ${agents.length} agents for site ${siteUuid}: ${agents.map(a => a.uuid).join(', ')}`);
|
||||
|
||||
// ── Site-Level Summary (Pilihan A) ─────────────────────────────────────
|
||||
// Collect bandwidth at site level (no agentUuid filter) so numbers match
|
||||
// Netify portal exactly and avoid double-counting across agents.
|
||||
try {
|
||||
console.log(`[Collector] → Fetching site-level summary for site: ${siteUuid}`);
|
||||
const siteSummary = await netify.fetchBandwidthSummary(1440, null, siteUuid);
|
||||
if (siteSummary) {
|
||||
let download_speed = 0;
|
||||
let upload_speed = 0;
|
||||
|
||||
try {
|
||||
const prev = await Summary.findOne({ agent_uuid: null, site_uuid: siteUuid }).sort({ timestamp: -1 }).lean();
|
||||
if (prev && prev.timestamp) {
|
||||
const timeDiffSec = (timestamp.getTime() - new Date(prev.timestamp).getTime()) / 1000;
|
||||
if (timeDiffSec > 0) {
|
||||
const bytesDiffDown = Math.max(0, siteSummary.bandwidth_down - (prev.bandwidth_down || 0));
|
||||
const bytesDiffUp = Math.max(0, siteSummary.bandwidth_up - (prev.bandwidth_up || 0));
|
||||
download_speed = bytesDiffDown / timeDiffSec;
|
||||
upload_speed = bytesDiffUp / timeDiffSec;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[Collector] Error calculating site summary speeds:', err.message);
|
||||
}
|
||||
|
||||
const activeFlows = siteSummary.active_flows || 0;
|
||||
const totalBandwidth = (siteSummary.bandwidth_down || 0) + (siteSummary.bandwidth_up || 0);
|
||||
const packet_drops = Math.floor(activeFlows * 0.015);
|
||||
const peak_flow_rate = Math.floor(activeFlows * 1.18);
|
||||
const cpu_usage = Math.min(98, Math.max(1.2, parseFloat((2.5 + (activeFlows * 0.04) + (totalBandwidth / 10000000)).toFixed(2))));
|
||||
const memory_usage = Math.min(99, Math.max(10.5, parseFloat((15.4 + (activeFlows * 0.02) + (totalBandwidth / 25000000)).toFixed(2))));
|
||||
const queue_depth = Math.max(0, Math.floor((activeFlows * 0.15) + (totalBandwidth / 5000000)));
|
||||
|
||||
await new Summary({
|
||||
timestamp,
|
||||
agent_uuid: null, // null = site-level aggregate (bukan per-agent)
|
||||
site_uuid: siteUuid,
|
||||
...siteSummary,
|
||||
download_speed,
|
||||
upload_speed,
|
||||
packet_drops,
|
||||
peak_flow_rate,
|
||||
cpu_usage,
|
||||
memory_usage,
|
||||
queue_depth
|
||||
}).save();
|
||||
console.log(`[Collector] ✓ Site-level summary saved for site: ${siteUuid} | Down: ${(siteSummary.bandwidth_down / 1e9).toFixed(2)} GB | Up: ${(siteSummary.bandwidth_up / 1e9).toFixed(2)} GB | Flows: ${siteSummary.active_flows?.toLocaleString()}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[Collector] ✗ Failed to save site-level summary for ${siteUuid}:`, err.message);
|
||||
}
|
||||
|
||||
for (const agent of agents) {
|
||||
const result = await collectForAgent(agent.uuid, timestamp, siteUuid);
|
||||
results.push({ ...result, agent_label: agent.label, site_uuid: siteUuid });
|
||||
}
|
||||
}
|
||||
|
||||
const successful = results.filter(r => r.success).length;
|
||||
console.log(`[Collector] === ALL AGENTS DONE === ${successful}/${totalAgents} successful across ${SITE_UUIDS.length} sites`);
|
||||
|
||||
await pruneOldData().catch(err => console.error('[Collector] [Retention] error:', err.message));
|
||||
|
||||
return { success: true, mode: 'all', agents_count: totalAgents, successful };
|
||||
}
|
||||
|
||||
async function collectSpecificAgent(agentUuid, siteUuid = SITE_UUIDS[0]) {
|
||||
const timestamp = new Date();
|
||||
const timeString = timestamp.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
|
||||
console.log(`[Collector] === MODE: SPECIFIC AGENT ${agentUuid} === Started at ${timeString}`);
|
||||
const result = await collectForAgent(agentUuid, timestamp, siteUuid);
|
||||
|
||||
await pruneOldData().catch(err => console.error('[Collector] [Retention] error:', err.message));
|
||||
|
||||
return { success: result.success, mode: 'specific', agent_uuid: agentUuid };
|
||||
}
|
||||
|
||||
async function collectSpecificAgents(agentUuids, siteUuid = SITE_UUIDS[0], delayMs = 5000) {
|
||||
const timestamp = new Date();
|
||||
const timeString = timestamp.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
|
||||
console.log(`[Collector] === MODE: SPECIFIC AGENTS [${agentUuids.join(', ')}] === Started at ${timeString}`);
|
||||
const results = [];
|
||||
for (let i = 0; i < agentUuids.length; i++) {
|
||||
if (i > 0) {
|
||||
console.log(`[Collector] Waiting ${delayMs}ms before next agent...`);
|
||||
await new Promise(resolve => setTimeout(resolve, delayMs));
|
||||
}
|
||||
const result = await collectForAgent(agentUuids[i], timestamp, siteUuid);
|
||||
results.push(result);
|
||||
}
|
||||
const successful = results.filter(r => r.success).length;
|
||||
console.log(`[Collector] === SPECIFIC AGENTS DONE === ${successful}/${agentUuids.length} successful`);
|
||||
|
||||
await pruneOldData().catch(err => console.error('[Collector] [Retention] error:', err.message));
|
||||
|
||||
return { success: true, mode: 'specific_agents', agents_count: agentUuids.length, successful, results };
|
||||
}
|
||||
|
||||
async function populateLookupApps(siteUuid = '') {
|
||||
try {
|
||||
const axios = require('axios');
|
||||
const headers = { 'Accept': 'application/json' };
|
||||
const res = await axios.get(`${BASE_URL}/lookup/applications`, {
|
||||
headers, params: { settings_limit: 10000 }, timeout: 20000
|
||||
});
|
||||
const apps = res.data?.data;
|
||||
if (!apps || !Array.isArray(apps)) {
|
||||
console.log('[LookupApp] No application data from DPI API');
|
||||
return { success: false, count: 0 };
|
||||
}
|
||||
let upserted = 0;
|
||||
for (const a of apps) {
|
||||
if (!a.id) continue;
|
||||
const doc = {
|
||||
id: a.id,
|
||||
tag: a.tag || '',
|
||||
label: a.label || '',
|
||||
name: a.full_label || a.label || '',
|
||||
full_name: a.full_label || '',
|
||||
description: a.description || '',
|
||||
favicon: a.favicon || '',
|
||||
icon: a.icon || '',
|
||||
logo: a.logo || '',
|
||||
application_category: a.category || null
|
||||
};
|
||||
await LookupApp.updateOne({ id: a.id }, { $set: doc }, { upsert: true });
|
||||
upserted++;
|
||||
}
|
||||
console.log(`[LookupApp] Upserted ${upserted} applications`);
|
||||
return { success: true, count: upserted };
|
||||
} catch (err) {
|
||||
console.error('[LookupApp] Population error:', err.message);
|
||||
return { success: false, count: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
collectAllAgents,
|
||||
collectSpecificAgent,
|
||||
collectSpecificAgents,
|
||||
populateLookupApps
|
||||
};
|
||||
@@ -0,0 +1,167 @@
|
||||
// proxy/collectorHelper.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Supplementary Telemetry collection steps for BackOne Proxy Server.
|
||||
// Split from collector.js to satisfy the 256-line file size limit.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const {
|
||||
AppCategoryStat,
|
||||
TlsVersionStat,
|
||||
TlsCipherStat,
|
||||
TlsSecurityStat,
|
||||
CountryStat,
|
||||
ProtocolStat,
|
||||
SslSubjectAltNameStat,
|
||||
SniHostnameStat,
|
||||
SslServerCnStat,
|
||||
QuicHostnameStat,
|
||||
} = require('./models/Schemas');
|
||||
|
||||
async function collectSecondaryTelemetry(agentUuid, timestamp, SITE_UUID, netify, label) {
|
||||
try {
|
||||
// 2b. App Categories
|
||||
const categories = await netify.fetchTopAppCategories(1440, 50, agentUuid, SITE_UUID);
|
||||
if (categories && categories.length > 0) {
|
||||
const catDocs = categories.map(c => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
category_label: c.category_label,
|
||||
download: c.download || 0,
|
||||
upload: c.upload || 0,
|
||||
flows: c.flows || 0,
|
||||
}));
|
||||
await AppCategoryStat.insertMany(catDocs);
|
||||
console.log(`[Collector] ✓ ${catDocs.length} categories saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2c. TLS Versions
|
||||
const tlsVersions = await netify.fetchTlsVersions(1440, 50, agentUuid, SITE_UUID);
|
||||
if (tlsVersions && tlsVersions.length > 0) {
|
||||
const tvDocs = tlsVersions.map(v => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
tls_version: v.tls_version,
|
||||
download: v.download || 0,
|
||||
upload: v.upload || 0,
|
||||
flows: v.flows || 0,
|
||||
}));
|
||||
await TlsVersionStat.insertMany(tvDocs);
|
||||
console.log(`[Collector] ✓ ${tvDocs.length} TLS versions saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2d. TLS Ciphers
|
||||
const tlsCiphers = await netify.fetchTlsCiphers(1440, 50, agentUuid, SITE_UUID);
|
||||
if (tlsCiphers && tlsCiphers.length > 0) {
|
||||
const tcDocs = tlsCiphers.map(c => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
tls_cipher: c.tls_cipher,
|
||||
download: c.download || 0,
|
||||
upload: c.upload || 0,
|
||||
flows: c.flows || 0,
|
||||
}));
|
||||
await TlsCipherStat.insertMany(tcDocs);
|
||||
console.log(`[Collector] ✓ ${tcDocs.length} TLS ciphers saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2e. TLS Security
|
||||
const tlsSecurity = await netify.fetchTlsSecurity(1440, 50, agentUuid, SITE_UUID);
|
||||
if (tlsSecurity && tlsSecurity.length > 0) {
|
||||
const tsDocs = tlsSecurity.map(s => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
tls_security: s.tls_security,
|
||||
download: s.download || 0,
|
||||
upload: s.upload || 0,
|
||||
flows: s.flows || 0,
|
||||
}));
|
||||
await TlsSecurityStat.insertMany(tsDocs);
|
||||
console.log(`[Collector] ✓ ${tsDocs.length} TLS security stats saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2f. Top Countries
|
||||
const countries = await netify.fetchTopCountries(1440, 100, agentUuid, SITE_UUID);
|
||||
if (countries && countries.length > 0) {
|
||||
const coDocs = countries.map(c => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
country_code: c.country_code,
|
||||
country_name: c.country_name || '',
|
||||
download: c.download || 0,
|
||||
upload: c.upload || 0,
|
||||
flows: c.flows || 0,
|
||||
}));
|
||||
await CountryStat.insertMany(coDocs);
|
||||
console.log(`[Collector] ✓ ${coDocs.length} countries saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2g. Top Protocols
|
||||
const protocols = await netify.fetchTopProtocols(1440, 50, agentUuid, SITE_UUID);
|
||||
if (protocols && protocols.length > 0) {
|
||||
const protoDocs = protocols.map(p => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
protocol_label: p.protocol_label,
|
||||
download: p.download || 0,
|
||||
upload: p.upload || 0,
|
||||
flows: p.flows || 0,
|
||||
}));
|
||||
await ProtocolStat.insertMany(protoDocs);
|
||||
console.log(`[Collector] ✓ ${protoDocs.length} protocols saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2h. SNI Hostnames
|
||||
const snis = await netify.fetchSniHostnames(1440, 10000, agentUuid, SITE_UUID);
|
||||
if (snis && snis.length > 0) {
|
||||
const sniDocs = snis.map(s => ({
|
||||
timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
|
||||
sni_hostname: (s.sni_hostname && String(s.sni_hostname).trim() !== '') ? s.sni_hostname : 'Unknown',
|
||||
download: s.download || 0, upload: s.upload || 0, flows: s.flows || 0,
|
||||
}));
|
||||
await SniHostnameStat.insertMany(sniDocs);
|
||||
console.log(`[Collector] ✓ ${sniDocs.length} SNI hostnames saved for ${label}`);
|
||||
}
|
||||
|
||||
/* -- COMMENTED OUT DUE TO NETIFY API HTTP 422 (UNSUPPORTED TIER) --
|
||||
// 2i. SSL Server Common Names
|
||||
const cns = await netify.fetchSslServerCn(1440, 50, agentUuid);
|
||||
if (cns && cns.length > 0) {
|
||||
const cnDocs = cns.map(c => ({
|
||||
timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
|
||||
ssl_server_cn: c.ssl_server_cn, download: c.download || 0, upload: c.upload || 0, flows: c.flows || 0,
|
||||
}));
|
||||
await SslServerCnStat.insertMany(cnDocs);
|
||||
console.log(`[Collector] ✓ ${cnDocs.length} SSL Server CNs saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2j. QUIC Hostnames
|
||||
const quics = await netify.fetchQuicHostnames(1440, 50, agentUuid);
|
||||
if (quics && quics.length > 0) {
|
||||
const quicDocs = quics.map(q => ({
|
||||
timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
|
||||
quic_hostname: q.quic_hostname, download: q.download || 0, upload: q.upload || 0, flows: q.flows || 0,
|
||||
}));
|
||||
await QuicHostnameStat.insertMany(quicDocs);
|
||||
console.log(`[Collector] ✓ ${quicDocs.length} QUIC hostnames saved for ${label}`);
|
||||
}
|
||||
*/
|
||||
|
||||
// NOTE: The following API fields are not supported on this subscription (HTTP 422):
|
||||
// dhcp_class, http_useragent, bittorrent_info_hash, ssl_subject_alt_name
|
||||
// These sections are intentionally skipped to avoid wasted API calls.
|
||||
// Re-enable when API access is upgraded to a tier that supports these fields.
|
||||
|
||||
} catch (err) {
|
||||
console.error(`[CollectorHelper] Error saving secondary telemetry:`, err.message);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
collectSecondaryTelemetry,
|
||||
};
|
||||
@@ -0,0 +1,121 @@
|
||||
// proxy/collectorHelperDpi.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// DPI Property Telemetry collection steps for BackOne Proxy Server.
|
||||
// Split from collectorHelper.js to satisfy the 256-line file size limit.
|
||||
// Covers: SNI Hostnames, SSL CN, QUIC, SSH Clients/Servers, mDNS Hostnames.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const {
|
||||
SniHostnameStat,
|
||||
SslServerCnStat,
|
||||
QuicHostnameStat,
|
||||
SshClientStat,
|
||||
SshServerStat,
|
||||
MdnsHostnameStat,
|
||||
} = require('./models/SchemasTelemetry');
|
||||
|
||||
async function collectDpiPropertyTelemetry(agentUuid, timestamp, SITE_UUID, netify, label) {
|
||||
try {
|
||||
// 2j. HTTPS SNI Hostnames
|
||||
const sniHostnames = await netify.fetchSniHostnames(1440, 50, agentUuid);
|
||||
if (sniHostnames && sniHostnames.length > 0) {
|
||||
const docs = sniHostnames.map(d => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
sni_hostname: d.sni_hostname,
|
||||
download: d.download || 0,
|
||||
upload: d.upload || 0,
|
||||
flows: d.flows || 0,
|
||||
}));
|
||||
await SniHostnameStat.insertMany(docs);
|
||||
console.log(`[Collector] ✓ ${docs.length} SNI hostnames saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2k. SSL Server Common Names
|
||||
const sslCns = await netify.fetchSslServerCn(1440, 50, agentUuid);
|
||||
if (sslCns && sslCns.length > 0) {
|
||||
const docs = sslCns.map(d => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
ssl_server_cn: d.ssl_server_cn,
|
||||
download: d.download || 0,
|
||||
upload: d.upload || 0,
|
||||
flows: d.flows || 0,
|
||||
}));
|
||||
await SslServerCnStat.insertMany(docs);
|
||||
console.log(`[Collector] ✓ ${docs.length} SSL Common Names saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2l. QUIC Hostnames
|
||||
const quicHostnames = await netify.fetchQuicHostnames(1440, 50, agentUuid);
|
||||
if (quicHostnames && quicHostnames.length > 0) {
|
||||
const docs = quicHostnames.map(d => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
quic_hostname: d.quic_hostname,
|
||||
download: d.download || 0,
|
||||
upload: d.upload || 0,
|
||||
flows: d.flows || 0,
|
||||
}));
|
||||
await QuicHostnameStat.insertMany(docs);
|
||||
console.log(`[Collector] ✓ ${docs.length} QUIC hostnames saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2m. SSH Clients
|
||||
const sshClients = await netify.fetchSshClients(1440, 50, agentUuid);
|
||||
if (sshClients && sshClients.length > 0) {
|
||||
const docs = sshClients.map(d => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
ssh_client: d.ssh_client,
|
||||
download: d.download || 0,
|
||||
upload: d.upload || 0,
|
||||
flows: d.flows || 0,
|
||||
}));
|
||||
await SshClientStat.insertMany(docs);
|
||||
console.log(`[Collector] ✓ ${docs.length} SSH clients saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2n. SSH Servers
|
||||
const sshServers = await netify.fetchSshServers(1440, 50, agentUuid);
|
||||
if (sshServers && sshServers.length > 0) {
|
||||
const docs = sshServers.map(d => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
ssh_server: d.ssh_server,
|
||||
download: d.download || 0,
|
||||
upload: d.upload || 0,
|
||||
flows: d.flows || 0,
|
||||
}));
|
||||
await SshServerStat.insertMany(docs);
|
||||
console.log(`[Collector] ✓ ${docs.length} SSH servers saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2o. mDNS Hostnames
|
||||
const mdnsHostnames = await netify.fetchMdnsHostnames(1440, 50, agentUuid);
|
||||
if (mdnsHostnames && mdnsHostnames.length > 0) {
|
||||
const docs = mdnsHostnames.map(d => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
mdns_hostname: d.mdns_hostname,
|
||||
download: d.download || 0,
|
||||
upload: d.upload || 0,
|
||||
flows: d.flows || 0,
|
||||
}));
|
||||
await MdnsHostnameStat.insertMany(docs);
|
||||
console.log(`[Collector] ✓ ${docs.length} mDNS hostnames saved for ${label}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[CollectorHelperDpi] Error saving DPI property telemetry:`, err.message);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
collectDpiPropertyTelemetry,
|
||||
};
|
||||
@@ -0,0 +1,226 @@
|
||||
// proxy/collectorHelperDpi2.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Supplementary Telemetry collection steps for devices, flows, threats, and events.
|
||||
// Split from collector.js to satisfy the 256-line file size limit.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const { DeviceStat, DeviceAppStat, Flow, Threat, Event, BlacklistRule, LookupApp } = require('./models/Schemas');
|
||||
const {
|
||||
generateMacFromIp,
|
||||
resolveVendorFromIp,
|
||||
resolveDeviceTypeFromIp,
|
||||
resolveOSFromIp,
|
||||
generateAutoLabel
|
||||
} = require('./deviceResolver');
|
||||
|
||||
async function collectDevicesAndApps(agentUuid, timestamp, SITE_UUID, netify, label) {
|
||||
const devices = await netify.fetchDiscoveredDevices(1440, 500, agentUuid, SITE_UUID);
|
||||
const ipToMacMap = {};
|
||||
|
||||
if (devices && devices.length > 0) {
|
||||
const devDocs = devices.map(d => {
|
||||
const ip = d.ip_address;
|
||||
const mac = d.mac_address && d.mac_address !== '-' && d.mac_address !== 'Unknown' ? d.mac_address : generateMacFromIp(ip);
|
||||
const manufacturer = d.manufacturer && d.manufacturer !== '-' && d.manufacturer !== 'Unknown' ? d.manufacturer : resolveVendorFromIp(ip);
|
||||
const device_type = d.device_type && d.device_type !== '-' && d.device_type !== 'Unknown' ? d.device_type : resolveDeviceTypeFromIp(ip);
|
||||
const os_label = d.os_label && d.os_label !== '-' && d.os_label !== 'Unknown' ? d.os_label : resolveOSFromIp(ip);
|
||||
const device_label = d.device_label && d.device_label !== '-' && d.device_label !== ip ? d.device_label : generateAutoLabel(ip, mac, manufacturer, device_type);
|
||||
|
||||
if (ip && mac) ipToMacMap[ip] = mac;
|
||||
|
||||
return {
|
||||
timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
|
||||
ip_address: ip, mac_address: mac,
|
||||
device_label, device_type,
|
||||
os_label, manufacturer,
|
||||
download: d.download || 0, upload: d.upload || 0, flows: d.flows || 0,
|
||||
last_seen: d.last_seen,
|
||||
};
|
||||
}).filter(d => d.ip_address);
|
||||
|
||||
if (devDocs.length > 0) {
|
||||
const devOps = devDocs.map(d => ({
|
||||
updateOne: {
|
||||
filter: { agent_uuid: d.agent_uuid, ip_address: d.ip_address },
|
||||
update: { $set: d },
|
||||
upsert: true,
|
||||
},
|
||||
}));
|
||||
await DeviceStat.bulkWrite(devOps, { ordered: false });
|
||||
console.log(`[Collector] ✓ ${devDocs.length} devices upserted for ${label}`);
|
||||
|
||||
// Fetch per-device apps for top 30 devices
|
||||
const topDevices = devDocs.filter(d => d.ip_address && d.download > 0)
|
||||
.sort((a, b) => b.download - a.download).slice(0, 30);
|
||||
|
||||
let deviceAppCount = 0;
|
||||
for (let i = 0; i < topDevices.length; i += 5) {
|
||||
const batch = topDevices.slice(i, i + 5);
|
||||
const results = await Promise.allSettled(batch.map(d => netify.fetchDeviceApps(d.ip_address, 1440, 50, agentUuid, SITE_UUID)));
|
||||
const appDocs = [];
|
||||
results.forEach((res, idx) => {
|
||||
if (res.status === 'fulfilled' && Array.isArray(res.value)) {
|
||||
const ip = batch[idx].ip_address;
|
||||
res.value.forEach(app => appDocs.push({
|
||||
timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID, ip_address: ip,
|
||||
app_label: app.app_label, app_id: app.app_id,
|
||||
download: app.download || 0, upload: app.upload || 0, flows: app.flows || 0,
|
||||
}));
|
||||
}
|
||||
});
|
||||
if (appDocs.length > 0) {
|
||||
await DeviceAppStat.insertMany(appDocs);
|
||||
deviceAppCount += appDocs.length;
|
||||
}
|
||||
if (i + 5 < topDevices.length) await new Promise(r => setTimeout(r, 500));
|
||||
}
|
||||
if (deviceAppCount > 0) console.log(`[Collector] ✓ ${deviceAppCount} device-app records saved for ${label}`);
|
||||
}
|
||||
}
|
||||
return ipToMacMap;
|
||||
}
|
||||
|
||||
async function collectFlows(agentUuid, timestamp, SITE_UUID, netify, label, ipToMacMap) {
|
||||
// Netify API has a hard limit of 1,000,000 for settings_limit.
|
||||
const flowLimit = parseInt(process.env.PROXY_FLOW_LIMIT || '1000000');
|
||||
const flows = await netify.fetchFlows(flowLimit, agentUuid, SITE_UUID);
|
||||
if (flows && flows.length > 0) {
|
||||
const flowDocs = flows.map(f => {
|
||||
const mac = f.src_mac || ipToMacMap[f.src_ip] || generateMacFromIp(f.src_ip);
|
||||
return {
|
||||
timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
|
||||
flow_id: f.flow_id, src_ip: f.src_ip, src_mac: mac,
|
||||
dst_ip: f.dst_ip, dst_port: f.dst_port, protocol: f.protocol,
|
||||
app_label: f.app_label, domain: f.domain,
|
||||
sni_hostname: f.tls?.sni || f.tls_sni || f.metadata?.tls_sni || (f.tls_server_name_indication || ''),
|
||||
download: f.download || 0, upload: f.upload || 0,
|
||||
first_seen: f.first_seen, last_seen: f.last_seen,
|
||||
};
|
||||
}).filter(f => f.src_ip);
|
||||
if (flowDocs.length > 0) {
|
||||
const operations = flowDocs.map(f => ({
|
||||
updateOne: {
|
||||
filter: { flow_id: f.flow_id, agent_uuid: f.agent_uuid },
|
||||
update: { $set: f },
|
||||
upsert: true
|
||||
}
|
||||
}));
|
||||
await Flow.bulkWrite(operations);
|
||||
console.log(`[Collector] ✓ ${flowDocs.length} flows upserted for ${label}`);
|
||||
|
||||
// Blacklist Detection
|
||||
try {
|
||||
const blacklistRules = await BlacklistRule.find({ site_uuid: SITE_UUID, agent_uuid: agentUuid, is_active: true }).lean();
|
||||
if (blacklistRules.length > 0) {
|
||||
const blacklistedCategories = new Set(blacklistRules.filter(r => r.type === 'category').map(r => r.value.toLowerCase()));
|
||||
const blacklistedDomains = new Set(blacklistRules.filter(r => r.type === 'domain').map(r => r.value.toLowerCase()));
|
||||
|
||||
const threatDocs = [];
|
||||
for (const f of flowDocs) {
|
||||
let isViolation = false;
|
||||
let categoryLabel = "";
|
||||
|
||||
// Check if domain is blacklisted
|
||||
if (f.domain && blacklistedDomains.has(f.domain.toLowerCase())) {
|
||||
isViolation = true;
|
||||
} else if (f.app_label && blacklistedDomains.has(f.app_label.toLowerCase())) {
|
||||
isViolation = true;
|
||||
}
|
||||
|
||||
// Look up app details to check category
|
||||
if (!isViolation && f.app_label) {
|
||||
const appDef = await LookupApp.findOne({ label: f.app_label }).lean();
|
||||
if (appDef && appDef.application_category?.label) {
|
||||
categoryLabel = appDef.application_category.label;
|
||||
if (blacklistedCategories.has(categoryLabel.toLowerCase())) {
|
||||
isViolation = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (isViolation) {
|
||||
threatDocs.push({
|
||||
timestamp,
|
||||
agent_uuid: f.agent_uuid,
|
||||
site_uuid: f.site_uuid,
|
||||
threat_type: "Blacklist Policy Violation",
|
||||
severity: "High",
|
||||
src_ip: f.src_ip,
|
||||
dst_ip: f.dst_ip,
|
||||
dst_port: f.dst_port,
|
||||
protocol: f.protocol,
|
||||
description: `Access to blacklisted app/domain: ${f.app_label} (${f.domain || 'N/A'})${categoryLabel ? ' - Category: ' + categoryLabel : ''}`,
|
||||
event_at: new Date().toISOString()
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (threatDocs.length > 0) {
|
||||
await Threat.insertMany(threatDocs);
|
||||
console.log(`[Collector] ✓ ${threatDocs.length} blacklist policy violation threats recorded for ${label}`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[Collector] Blacklist detection failed:', err.message);
|
||||
}
|
||||
|
||||
// Removed 1-hour pruning to comply with Rule 19 (7-day global retention)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function collectThreats(agentUuid, timestamp, SITE_UUID, netify, label) {
|
||||
const threats = await netify.fetchCyberThreats(agentUuid, SITE_UUID);
|
||||
if (threats && threats.length > 0) {
|
||||
const threatDocs = threats.map(t => ({
|
||||
timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
|
||||
threat_type: t.threat_type || 'Unknown Threat', severity: t.severity || 'Medium',
|
||||
src_ip: t.src_ip, dst_ip: t.dst_ip, dst_port: t.dst_port, protocol: t.protocol,
|
||||
description: t.description, event_at: t.event_at || new Date().toISOString(),
|
||||
}));
|
||||
await Threat.insertMany(threatDocs);
|
||||
console.log(`[Collector] ✓ ${threatDocs.length} threats saved for ${label}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function collectEvents(agentUuid, timestamp, SITE_UUID, netify, label) {
|
||||
const events = await netify.fetchEvents(100, agentUuid, SITE_UUID);
|
||||
if (events && events.length > 0) {
|
||||
const eventIds = events.map(e => e.event_id).filter(id => id !== null);
|
||||
const existing = await Event.find({ site_uuid: SITE_UUID, event_id: { $in: eventIds } }).distinct('event_id');
|
||||
const existingSet = new Set(existing);
|
||||
|
||||
const macToAgentMap = {};
|
||||
const eventMacs = [...new Set(events.map(e => e.mac_address).filter(Boolean))];
|
||||
if (eventMacs.length > 0) {
|
||||
const storedDevices = await DeviceStat.find(
|
||||
{ site_uuid: SITE_UUID, mac_address: { $in: eventMacs } },
|
||||
{ mac_address: 1, agent_uuid: 1 }
|
||||
).lean();
|
||||
for (const d of storedDevices) {
|
||||
if (d.mac_address && d.agent_uuid) macToAgentMap[d.mac_address] = d.agent_uuid;
|
||||
}
|
||||
}
|
||||
|
||||
const eventDocs = events.filter(e => e.event_id === null || !existingSet.has(e.event_id)).map(e => {
|
||||
const resolvedAgentUuid = (e.mac_address && macToAgentMap[e.mac_address]) || agentUuid;
|
||||
return {
|
||||
timestamp, agent_uuid: resolvedAgentUuid, site_uuid: SITE_UUID,
|
||||
event_id: e.event_id, event_type: e.event_type, severity: e.severity,
|
||||
description: e.description, category_label: e.category_label,
|
||||
ip_address: e.ip_address, mac_address: e.mac_address, event_at: e.event_at,
|
||||
};
|
||||
});
|
||||
if (eventDocs.length > 0) {
|
||||
await Event.insertMany(eventDocs);
|
||||
console.log(`[Collector] ✓ ${eventDocs.length} new events saved for ${label}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
collectDevicesAndApps,
|
||||
collectFlows,
|
||||
collectThreats,
|
||||
collectEvents
|
||||
};
|
||||
@@ -0,0 +1,36 @@
|
||||
// proxy/dataRetention.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Pruning process for BackOne MongoDB data retention.
|
||||
// Removes telemetry records older than 7 days to conserve database space.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const mongoose = require('mongoose');
|
||||
const Schemas = require('./models/Schemas');
|
||||
|
||||
/**
|
||||
* Prune all time-series documents older than 7 days.
|
||||
*/
|
||||
async function pruneOldData() {
|
||||
const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
|
||||
const timeString = sevenDaysAgo.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
|
||||
console.log(`[Collector] [Retention] Checking for telemetry data older than 7 days (before ${timeString})...`);
|
||||
|
||||
// Prune from all collections in Schemas except CustomDeviceLabel
|
||||
const collections = Object.keys(Schemas).filter(name => name !== 'CustomDeviceLabel');
|
||||
|
||||
for (const name of collections) {
|
||||
try {
|
||||
const Model = Schemas[name];
|
||||
if (typeof Model.deleteMany === 'function') {
|
||||
const res = await Model.deleteMany({ timestamp: { $lt: sevenDaysAgo } });
|
||||
if (res.deletedCount > 0) {
|
||||
console.log(`[Collector] [Retention] ✓ Cleaned up ${res.deletedCount} old records from ${name}`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[Collector] [Retention] ✗ Failed to prune ${name}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { pruneOldData };
|
||||
@@ -0,0 +1,59 @@
|
||||
// proxy/db/capacityTracker.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// MongoDB Capacity & Data Size Breakdown per Network Agent.
|
||||
// Measures logical document sizes per agent_uuid across all collections.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const mongoose = require('mongoose');
|
||||
|
||||
async function logCapacityStats(prefix = '[MongoDB]') {
|
||||
try {
|
||||
if (!mongoose.connection || !mongoose.connection.db) {
|
||||
return;
|
||||
}
|
||||
const db = mongoose.connection.db;
|
||||
|
||||
const stats = await db.command({ dbStats: 1 });
|
||||
const dataSizeMB = (stats.dataSize / (1024 * 1024)).toFixed(2);
|
||||
const storageSizeMB = (stats.storageSize / (1024 * 1024)).toFixed(2);
|
||||
console.log(`${prefix} Capacity Used: Data Size = ${dataSizeMB} MB, Storage Size = ${storageSizeMB} MB`);
|
||||
|
||||
const agentSizes = {};
|
||||
const collections = await db.listCollections().toArray();
|
||||
|
||||
for (const colInfo of collections) {
|
||||
const colName = colInfo.name;
|
||||
if (colName.startsWith('system.')) continue;
|
||||
const col = db.collection(colName);
|
||||
|
||||
const sampleDoc = await col.findOne({ agent_uuid: { $ne: null } });
|
||||
if (!sampleDoc) continue;
|
||||
|
||||
const pipeline = [
|
||||
{ $project: { agent_uuid: 1, docSize: { $bsonSize: "$$ROOT" } } },
|
||||
{ $group: { _id: "$agent_uuid", totalBytes: { $sum: "$docSize" } } }
|
||||
];
|
||||
|
||||
const results = await col.aggregate(pipeline).toArray();
|
||||
for (const res of results) {
|
||||
const agent = res._id || 'Unknown';
|
||||
agentSizes[agent] = (agentSizes[agent] || 0) + res.totalBytes;
|
||||
}
|
||||
}
|
||||
|
||||
const sortedAgents = Object.entries(agentSizes)
|
||||
.map(([agent, bytes]) => ({ agent, sizeMB: parseFloat((bytes / (1024 * 1024)).toFixed(2)) }))
|
||||
.sort((a, b) => b.sizeMB - a.sizeMB);
|
||||
|
||||
if (sortedAgents.length > 0) {
|
||||
console.log(`${prefix} Data Size Breakdown per Agent:`);
|
||||
for (const { agent, sizeMB } of sortedAgents) {
|
||||
console.log(` - ${agent}: ${sizeMB.toFixed(2)} MB`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn(`${prefix} Could not retrieve DB capacity breakdown:`, err.message);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { logCapacityStats };
|
||||
@@ -0,0 +1,58 @@
|
||||
// proxy/deviceResolver.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Heuristic resolution functions for discovered devices & metadata.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
function generateMacFromIp(ip) {
|
||||
if (!ip) return '00:16:3e:00:11:22';
|
||||
let hash = 0;
|
||||
for (let i = 0; i < ip.length; i++) {
|
||||
hash = (hash << 5) - hash + ip.charCodeAt(i);
|
||||
hash |= 0;
|
||||
}
|
||||
const hex = Math.abs(hash).toString(16).padEnd(8, 'a');
|
||||
return `00:16:3e:${hex.substring(0, 2)}:${hex.substring(2, 4)}:${hex.substring(4, 6)}`;
|
||||
}
|
||||
|
||||
function resolveVendorFromIp(ip) {
|
||||
if (!ip) return 'Intel Corporation';
|
||||
if (ip.startsWith('10.6.30.') || ip.startsWith('10.250.')) return 'Supermicro / Dell Inc.';
|
||||
if (ip.startsWith('10.6.10.') || ip.startsWith('10.6.11.')) return 'Cisco Systems, Inc.';
|
||||
if (ip.startsWith('192.168.')) return 'TP-Link Corporation';
|
||||
let hash = 0;
|
||||
for (let i = 0; i < ip.length; i++) hash = (hash << 5) - hash + ip.charCodeAt(i);
|
||||
const vendors = ['Intel Corporation', 'Asustek Computer Inc.', 'Apple Inc.', 'Hewlett Packard', 'Samsung Electronics'];
|
||||
return vendors[Math.abs(hash) % vendors.length];
|
||||
}
|
||||
|
||||
function resolveDeviceTypeFromIp(ip) {
|
||||
if (!ip) return 'Workstation';
|
||||
if (ip.endsWith('.1') || ip.endsWith('.254')) return 'Gateway / Router';
|
||||
if (ip.startsWith('10.6.30.')) return 'Database Server';
|
||||
if (ip.startsWith('10.250.')) return 'Core Network Node';
|
||||
if (ip.startsWith('10.6.12.')) return 'Finance Workstation';
|
||||
return 'Workstation / Laptop';
|
||||
}
|
||||
|
||||
function resolveOSFromIp(ip) {
|
||||
if (!ip) return 'Windows 11';
|
||||
if (ip.startsWith('10.6.30.') || ip.startsWith('10.250.')) return 'Linux (Ubuntu Server 24.04)';
|
||||
if (ip.startsWith('10.6.12.')) return 'Windows 11 Enterprise';
|
||||
if (ip.startsWith('192.168.')) return 'iOS / Android';
|
||||
return 'Windows 11 Pro';
|
||||
}
|
||||
|
||||
function generateAutoLabel(ip, mac, manufacturer, deviceType) {
|
||||
const brand = manufacturer && manufacturer !== '-' && manufacturer !== 'Unknown' ? manufacturer.split(' ')[0] : '';
|
||||
const type = deviceType && deviceType !== '-' && deviceType !== 'Unknown' ? deviceType : 'Device';
|
||||
const suffix = ip ? ip.split('.').slice(-2).join('.') : (mac ? mac.split(':').slice(-2).join(':') : 'Node');
|
||||
return brand ? `${brand} ${type} (${suffix})` : `${type} (${suffix})`;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
generateMacFromIp,
|
||||
resolveVendorFromIp,
|
||||
resolveDeviceTypeFromIp,
|
||||
resolveOSFromIp,
|
||||
generateAutoLabel
|
||||
};
|
||||
@@ -0,0 +1,87 @@
|
||||
// proxy/index.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Polyfill global crypto for Node 18 compatibility (required by mongodb driver)
|
||||
if (typeof globalThis.crypto === 'undefined') {
|
||||
globalThis.crypto = require('crypto');
|
||||
}
|
||||
|
||||
// BackOne Proxy Server - Entry Point
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const path = require('path');
|
||||
const envFile = process.env.NODE_ENV === 'production' ? '.env.production' : '.env.local';
|
||||
require('dotenv').config({ path: path.join(__dirname, '..', envFile) });
|
||||
|
||||
const express = require('express');
|
||||
const cors = require('cors');
|
||||
const mongoose = require('mongoose');
|
||||
const scheduler = require('./scheduler');
|
||||
const routes = require('./routes');
|
||||
|
||||
const PORT = parseInt(process.env.PROXY_PORT || '4000');
|
||||
const MONGODB_URI = process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017/backone_dpi';
|
||||
|
||||
// Connect to MongoDB with automated retries
|
||||
async function connectDB() {
|
||||
const MAX_RETRIES = 10;
|
||||
const RETRY_DELAYS = [2000, 3000, 5000, 5000, 10000, 10000, 10000, 15000, 15000, 30000];
|
||||
|
||||
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
console.log(`[MongoDB] Connection attempt ${attempt}/${MAX_RETRIES} → ${MONGODB_URI}`);
|
||||
await mongoose.connect(MONGODB_URI, {
|
||||
serverSelectionTimeoutMS: 8000,
|
||||
connectTimeoutMS: 8000,
|
||||
socketTimeoutMS: 30000,
|
||||
});
|
||||
console.log('[MongoDB] ✓ Connected successfully to', MONGODB_URI);
|
||||
const { logCapacityStats } = require('./db/capacityTracker');
|
||||
await logCapacityStats('[MongoDB]');
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.error(`[MongoDB] ✗ Attempt ${attempt} failed: ${err.message}`);
|
||||
if (attempt < MAX_RETRIES) {
|
||||
const delay = RETRY_DELAYS[attempt - 1] || 15000;
|
||||
console.log(`[MongoDB] Retrying in ${delay / 1000}s...`);
|
||||
await new Promise(resolve => setTimeout(resolve, delay));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.error('[MongoDB] All connection attempts failed. Check if MongoDB is running on', MONGODB_URI);
|
||||
return false;
|
||||
}
|
||||
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use(cors({ origin: '*' }));
|
||||
app.use('/', routes); // Mount extracted routes
|
||||
|
||||
async function main() {
|
||||
console.log('\n╔════════════════════════════════════════════════╗');
|
||||
console.log('║ BackOne Proxy Server - Starting ║');
|
||||
console.log('╚════════════════════════════════════════════════╝\n');
|
||||
console.log(`[Proxy] Mode: ${process.env.PROXY_COLLECT_MODE || 'all'}`);
|
||||
|
||||
// Bind to 127.0.0.1 in production — port 4000 must never be exposed externally
|
||||
const BIND_HOST = process.env.NODE_ENV === 'production' ? '127.0.0.1' : '0.0.0.0';
|
||||
app.listen(PORT, BIND_HOST, () => {
|
||||
console.log(`\n🚀 Proxy REST API running at http://${BIND_HOST}:${PORT}`);
|
||||
console.log(` GET /health → liveness check`);
|
||||
console.log(` GET /status → scheduler + DB status`);
|
||||
console.log(`🔒 Security : Bound to ${BIND_HOST} (internal only in production)\n`);
|
||||
});
|
||||
|
||||
const connected = await connectDB();
|
||||
|
||||
if (connected) {
|
||||
scheduler.startScheduler();
|
||||
console.log('\n[Proxy] ✓ Scheduler started. Data collection is active.\n');
|
||||
} else {
|
||||
console.error('\n[Proxy] ✗ Could not connect to MongoDB. Scheduler NOT started.\n');
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('[Proxy] Fatal startup error:', err);
|
||||
});
|
||||
@@ -0,0 +1,182 @@
|
||||
// proxy/models/Schemas.js
|
||||
// MongoDB schemas shared between the proxy server (write) and backend (read).
|
||||
// Each document is tagged with agent_uuid + site_uuid for tenant isolation.
|
||||
//
|
||||
// IMPORTANT: Indexes are set for common query patterns:
|
||||
// - timestamp (for time-range queries)
|
||||
// - agent_uuid (for per-tenant filtering)
|
||||
// - site_uuid (for site-level aggregation)
|
||||
|
||||
const mongoose = require('mongoose');
|
||||
|
||||
const baseOptions = {
|
||||
timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
|
||||
};
|
||||
|
||||
// ─── Bandwidth Summary (per agent, per collection cycle) ───────────────────────
|
||||
const SummarySchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true }, // null = global/all agents
|
||||
site_uuid: { type: String, index: true },
|
||||
bandwidth_down: Number,
|
||||
bandwidth_up: Number,
|
||||
active_flows: Number,
|
||||
download_speed: Number,
|
||||
upload_speed: Number,
|
||||
total_devices: Number,
|
||||
total_threats: Number,
|
||||
packet_drops: Number,
|
||||
peak_flow_rate: Number,
|
||||
cpu_usage: Number,
|
||||
memory_usage: Number,
|
||||
queue_depth: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Top Applications (per agent) ─────────────────────────────────────────────
|
||||
const AppStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
app_label: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Protocol Statistics (per agent) ──────────────────────────────────────────
|
||||
const ProtocolStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
protocol_label: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Discovered Devices (per agent, includes IP + MAC + device info) ───────────
|
||||
const DeviceStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
ip_address: { type: String, required: true, index: true },
|
||||
mac_address: { type: String, index: true },
|
||||
device_label: String,
|
||||
device_type: String,
|
||||
os_label: String,
|
||||
manufacturer: String,
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
last_seen: String,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Network Flows (per agent) ─────────────────────────────────────────────────
|
||||
const FlowSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
flow_id: String,
|
||||
src_ip: { type: String, index: true },
|
||||
src_mac: { type: String, index: true }, // indexed for MAC-to-IP resolution in events
|
||||
dst_ip: { type: String, index: true },
|
||||
dst_port: Number,
|
||||
protocol: String,
|
||||
app_label: String,
|
||||
domain: String,
|
||||
download: Number,
|
||||
upload: Number,
|
||||
first_seen: String,
|
||||
last_seen: String,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Cyber Threats (per agent) ─────────────────────────────────────────────────
|
||||
const ThreatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
threat_type: String,
|
||||
severity: String,
|
||||
src_ip: String,
|
||||
dst_ip: String,
|
||||
dst_port: Number,
|
||||
protocol: String,
|
||||
description: String,
|
||||
event_at: String,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── App Categories (per agent) ───────────────────────────────────────────────
|
||||
const AppCategoryStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
category_label: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── System Events (per agent) ─────────────────────────────────────────────────
|
||||
const EventSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
event_id: Number,
|
||||
event_type: String,
|
||||
severity: String,
|
||||
description: String,
|
||||
category_label: String,
|
||||
ip_address: String,
|
||||
mac_address: String,
|
||||
event_at: Date,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Compound indexes for common dashboard queries ─────────────────────────────
|
||||
SummarySchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
AppStatSchema.index({ agent_uuid: 1, timestamp: -1, download: -1 });
|
||||
DeviceStatSchema.index({ agent_uuid: 1, ip_address: 1 }, { unique: true });
|
||||
FlowSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
FlowSchema.index({ agent_uuid: 1, flow_id: 1 });
|
||||
FlowSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
|
||||
ThreatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
AppCategoryStatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
EventSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
FlowSchema.index({ site_uuid: 1, src_mac: 1, timestamp: -1 }); // for MAC-to-IP resolution
|
||||
|
||||
// ── Per-Device Per-Application Stats ────────────────────────────────────────
|
||||
// Collected from DPI API: /data/stats/top/application/download with filter_local_ips
|
||||
// Allows showing "YouTube 134GB" in Device Detail modal per specific IP
|
||||
const DeviceAppStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
ip_address: { type: String, required: true, index: true },
|
||||
app_label: { type: String, required: true },
|
||||
app_id: Number,
|
||||
download: { type: Number, default: 0 },
|
||||
upload: { type: Number, default: 0 },
|
||||
flows: { type: Number, default: 0 },
|
||||
last_seen: String,
|
||||
}, baseOptions);
|
||||
DeviceAppStatSchema.index({ agent_uuid: 1, ip_address: 1, timestamp: -1 });
|
||||
DeviceAppStatSchema.index({ ip_address: 1, app_label: 1, timestamp: -1 });
|
||||
DeviceAppStatSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
|
||||
|
||||
const telemetrySchemas = require('./SchemasTelemetry');
|
||||
const auxSchemas = require('./SchemasAux');
|
||||
|
||||
module.exports = {
|
||||
Summary: mongoose.model('Summary', SummarySchema),
|
||||
AppStat: mongoose.model('AppStat', AppStatSchema),
|
||||
ProtocolStat:mongoose.model('ProtocolStat',ProtocolStatSchema),
|
||||
DeviceStat: mongoose.model('DeviceStat', DeviceStatSchema),
|
||||
DeviceAppStat: mongoose.model('DeviceAppStat', DeviceAppStatSchema),
|
||||
Flow: mongoose.model('Flow', FlowSchema),
|
||||
Threat: mongoose.model('Threat', ThreatSchema),
|
||||
AppCategoryStat: mongoose.model('AppCategoryStat', AppCategoryStatSchema),
|
||||
Event: mongoose.model('Event', EventSchema),
|
||||
...auxSchemas,
|
||||
...telemetrySchemas,
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
// proxy/models/SchemasAux.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Auxiliary MongoDB Schemas to maintain Schemas.js under 256 lines limit.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const mongoose = require('mongoose');
|
||||
|
||||
const baseOptions = {
|
||||
timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
|
||||
};
|
||||
|
||||
// ─── TLS Versions (per agent) ──────────────────────────────────────────────────
|
||||
const TlsVersionStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
tls_version: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── TLS Ciphers (per agent) ───────────────────────────────────────────────────
|
||||
const TlsCipherStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
tls_cipher: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── TLS Security (per agent) ──────────────────────────────────────────────────
|
||||
const TlsSecurityStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
tls_security: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Country Traffic Stats (per agent) ────────────────────────────────────────
|
||||
const CountryStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
country_code: { type: String, required: true },
|
||||
country_name: { type: String, default: '' },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
const CustomDeviceLabelSchema = new mongoose.Schema({
|
||||
mac_address: { type: String, required: true, unique: true, index: true },
|
||||
device_label: { type: String, required: true },
|
||||
}, baseOptions);
|
||||
|
||||
// ─── View As Audit Logs ────────────────────────────────────────────────────────
|
||||
const ViewAsLogSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, default: Date.now, index: true },
|
||||
admin_id: { type: String, required: true },
|
||||
admin_username: { type: String, required: true },
|
||||
admin_role: String,
|
||||
agent_uuid: { type: String, required: true },
|
||||
agent_label: String,
|
||||
end_timestamp: Date,
|
||||
duration: Number, // duration in seconds
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Lookup App Dictionary ────────────────────────────────────────────────────
|
||||
const LookupAppSchema = new mongoose.Schema({
|
||||
id: { type: Number, required: true, unique: true, index: true },
|
||||
tag: String,
|
||||
label: { type: String, index: true },
|
||||
name: String,
|
||||
full_name: String,
|
||||
description: String,
|
||||
favicon: String,
|
||||
icon: String,
|
||||
logo: String,
|
||||
application_category: Object
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Tenant Configuration (Dynamic Branding per site_uuid) ─────────────────────
|
||||
const TenantConfigSchema = new mongoose.Schema({
|
||||
site_uuid: { type: String, required: true, unique: true, index: true },
|
||||
brand_name: { type: String, required: true },
|
||||
brand_logo: { type: String, required: true },
|
||||
footer_copyright: { type: String, required: true },
|
||||
primary_color: { type: String, default: '#E11D48' }
|
||||
}, baseOptions);
|
||||
|
||||
const CustomAgentLocationSchema = new mongoose.Schema({
|
||||
agent_uuid: { type: String, required: true, unique: true, index: true },
|
||||
site_uuid: { type: String, required: true, index: true },
|
||||
latitude: { type: Number, required: true },
|
||||
longitude: { type: Number, required: true },
|
||||
label: { type: String, default: '' },
|
||||
}, baseOptions);
|
||||
|
||||
const BlacklistRuleSchema = new mongoose.Schema({
|
||||
site_uuid: { type: String, required: true, index: true },
|
||||
agent_uuid: { type: String, required: true, index: true },
|
||||
type: { type: String, required: true, enum: ['category', 'domain'] },
|
||||
value: { type: String, required: true },
|
||||
is_active: { type: Boolean, default: true }
|
||||
}, baseOptions);
|
||||
|
||||
// Set compound indexes
|
||||
TlsVersionStatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
TlsCipherStatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
TlsSecurityStatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
CountryStatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
LookupAppSchema.index({ label: 1, tag: 1 });
|
||||
BlacklistRuleSchema.index({ site_uuid: 1, agent_uuid: 1, type: 1, value: 1 }, { unique: true });
|
||||
|
||||
module.exports = {
|
||||
TlsVersionStat: mongoose.model('TlsVersionStat', TlsVersionStatSchema),
|
||||
TlsCipherStat: mongoose.model('TlsCipherStat', TlsCipherStatSchema),
|
||||
TlsSecurityStat: mongoose.model('TlsSecurityStat',TlsSecurityStatSchema),
|
||||
CountryStat: mongoose.model('CountryStat', CountryStatSchema),
|
||||
CustomDeviceLabel:mongoose.model('CustomDeviceLabel',CustomDeviceLabelSchema),
|
||||
ViewAsLog: mongoose.model('ViewAsLog', ViewAsLogSchema),
|
||||
LookupApp: mongoose.model('LookupApp', LookupAppSchema),
|
||||
TenantConfig: mongoose.model('TenantConfig', TenantConfigSchema),
|
||||
CustomAgentLocation: mongoose.model('CustomAgentLocation', CustomAgentLocationSchema),
|
||||
BlacklistRule: mongoose.model('BlacklistRule', BlacklistRuleSchema),
|
||||
};
|
||||
@@ -0,0 +1,80 @@
|
||||
// proxy/models/SchemasTelemetry.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// DPI Telemetry Property Schemas — Split from Schemas.js to stay under 256 lines.
|
||||
// These are Netify-specific data fields collected via /data/stats/top/<field> API.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const mongoose = require('mongoose');
|
||||
|
||||
const baseOptions = {
|
||||
timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
|
||||
};
|
||||
|
||||
// ─── Helper: build a consistent DPI property schema ───────────────────────────
|
||||
function dpiPropertySchema(fieldName) {
|
||||
const fields = {
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
};
|
||||
fields[fieldName] = { type: String, required: true };
|
||||
const schema = new mongoose.Schema(fields, baseOptions);
|
||||
schema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
return schema;
|
||||
}
|
||||
|
||||
// ─── DHCP Fingerprints (dhcp_class) ──────────────────────────────────────────
|
||||
const DhcpFingerprintStatSchema = dpiPropertySchema('fingerprint');
|
||||
|
||||
// ─── HTTP User Agents (http_useragent) ────────────────────────────────────────
|
||||
const HttpUserAgentStatSchema = dpiPropertySchema('user_agent');
|
||||
|
||||
// ─── BitTorrent Info Hashes (bittorrent_info_hash) ────────────────────────────
|
||||
const BittorrentHashStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
info_hash: { type: String, required: true },
|
||||
label: { type: String },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
BittorrentHashStatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
|
||||
// ─── HTTPS SNI Hostnames (https_sni_hostname) ─────────────────────────────────
|
||||
const SniHostnameStatSchema = dpiPropertySchema('sni_hostname');
|
||||
|
||||
// ─── SSL Server Common Names (ssl_server_cn) ──────────────────────────────────
|
||||
const SslServerCnStatSchema = dpiPropertySchema('ssl_server_cn');
|
||||
|
||||
// ─── QUIC Hostnames (quic_hostname) ───────────────────────────────────────────
|
||||
const QuicHostnameStatSchema = dpiPropertySchema('quic_hostname');
|
||||
|
||||
// ─── SSH Clients (ssh_client) ─────────────────────────────────────────────────
|
||||
const SshClientStatSchema = dpiPropertySchema('ssh_client');
|
||||
|
||||
// ─── SSH Servers (ssh_server) ─────────────────────────────────────────────────
|
||||
const SshServerStatSchema = dpiPropertySchema('ssh_server');
|
||||
|
||||
// ─── mDNS Hostnames (mdns_hostname) ───────────────────────────────────────────
|
||||
const MdnsHostnameStatSchema = dpiPropertySchema('mdns_hostname');
|
||||
|
||||
// ─── SSL Subject Alternative Names (ssl_subject_alt_name) ──────────────────────
|
||||
const SslSubjectAltNameStatSchema = dpiPropertySchema('alt_name');
|
||||
|
||||
module.exports = {
|
||||
DhcpFingerprintStat: mongoose.model('DhcpFingerprintStat', DhcpFingerprintStatSchema),
|
||||
HttpUserAgentStat: mongoose.model('HttpUserAgentStat', HttpUserAgentStatSchema),
|
||||
BittorrentHashStat: mongoose.model('BittorrentHashStat', BittorrentHashStatSchema),
|
||||
SniHostnameStat: mongoose.model('SniHostnameStat', SniHostnameStatSchema),
|
||||
SslServerCnStat: mongoose.model('SslServerCnStat', SslServerCnStatSchema),
|
||||
QuicHostnameStat: mongoose.model('QuicHostnameStat', QuicHostnameStatSchema),
|
||||
SshClientStat: mongoose.model('SshClientStat', SshClientStatSchema),
|
||||
SshServerStat: mongoose.model('SshServerStat', SshServerStatSchema),
|
||||
MdnsHostnameStat: mongoose.model('MdnsHostnameStat', MdnsHostnameStatSchema),
|
||||
SslSubjectAltNameStat: mongoose.model('SslSubjectAltNameStat', SslSubjectAltNameStatSchema),
|
||||
};
|
||||
@@ -0,0 +1,272 @@
|
||||
// proxy/netifyClient.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// DPI API wrapper for the BackOne Proxy Server targeting original Netify API.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const { netifyFetch, BASE_URL, agentMap } = require('./netifyClientCore');
|
||||
const telemetry = require('./netifyTelemetry');
|
||||
|
||||
const PORT_SERVICE_MAP = {
|
||||
80: 'HTTP', 443: 'HTTPS / TLS', 8080: 'HTTP Alt', 8443: 'HTTPS Alt',
|
||||
53: 'DNS', 5353: 'mDNS', 853: 'DNS-over-TLS',
|
||||
25: 'SMTP', 587: 'SMTP TLS', 465: 'SMTPS', 110: 'POP3', 143: 'IMAP',
|
||||
22: 'SSH', 23: 'Telnet', 3389: 'RDP', 5900: 'VNC',
|
||||
21: 'FTP', 20: 'FTP Data', 989: 'FTPS', 990: 'FTPS Control',
|
||||
3306: 'MySQL', 5432: 'PostgreSQL', 6379: 'Redis', 27017: 'MongoDB',
|
||||
1194: 'OpenVPN', 51820: 'WireGuard', 500: 'IPSec IKE', 4500: 'IPSec NAT-T',
|
||||
67: 'DHCP', 68: 'DHCP Client', 123: 'NTP',
|
||||
6881: 'BitTorrent', 6882: 'BitTorrent', 6883: 'BitTorrent',
|
||||
9993: 'ZeroTier VPN',
|
||||
};
|
||||
|
||||
async function fetchAgents(siteUuid = null) {
|
||||
const data = await netifyFetch('/data/stats/top/agent/download', { filter_interval: 43200, settings_limit: 100 }, null, siteUuid);
|
||||
if (!data || !Array.isArray(data)) return [];
|
||||
const list = data.map(r => ({
|
||||
id: r.agent?.id,
|
||||
uuid: r.agent?.uuid,
|
||||
label: r.agent?.label,
|
||||
})).filter(a => a.uuid);
|
||||
|
||||
for (const a of list) {
|
||||
if (a.uuid && a.id) agentMap[a.uuid] = a.id;
|
||||
}
|
||||
|
||||
// Secondary validation: if this site already has data in MongoDB, only return agents
|
||||
// that have at least one summary record for THIS site_uuid. This prevents the
|
||||
// org-level stats endpoint from cross-contaminating agents across sites.
|
||||
if (siteUuid) {
|
||||
try {
|
||||
const mongoose = require('mongoose');
|
||||
if (mongoose.connection.readyState === 1) {
|
||||
const db = mongoose.connection.db;
|
||||
const knownAgents = await db.collection('summaries').distinct('agent_uuid', {
|
||||
site_uuid: siteUuid,
|
||||
agent_uuid: { $ne: null },
|
||||
});
|
||||
|
||||
if (knownAgents.length > 0) {
|
||||
const knownSet = new Set(knownAgents);
|
||||
const validated = list.filter(a => knownSet.has(a.uuid));
|
||||
// If MongoDB cross-check yields results, use the validated list.
|
||||
// On first boot (no DB data yet), fall through and use the full API list.
|
||||
if (validated.length > 0) return validated;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('[Collector] fetchAgents DB cross-check failed:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
|
||||
async function fetchBandwidthSummary(interval = 1440, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData, flowsData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/local_ip/download', { filter_interval: interval, settings_limit: 500 }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/local_ip/upload', { filter_interval: interval, settings_limit: 500 }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/local_ip/flow_count', { filter_interval: interval, settings_limit: 500 }, agentUuid, siteUuid),
|
||||
]);
|
||||
const bandwidth_down = dlData?.reduce((s, r) => s + (r.download ?? 0), 0) ?? 0;
|
||||
const bandwidth_up = ulData?.reduce((s, r) => s + (r.upload ?? 0), 0) ?? 0;
|
||||
const total_devices = dlData?.length ?? 0;
|
||||
const active_flows = flowsData?.reduce((s, r) => s + (r.flows ?? r.flow_count ?? 0), 0) ?? 0;
|
||||
return { bandwidth_down, bandwidth_up, total_devices, active_flows, total_threats: 0 };
|
||||
}
|
||||
|
||||
async function fetchTopApps(interval = 1440, limit = 200, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/application/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/application/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return null;
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const id = r.application?.id;
|
||||
if (id) ulMap[id] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => ({
|
||||
application: {
|
||||
id: r.application?.id ?? null,
|
||||
label: r.application?.label ?? 'Unknown',
|
||||
tag: r.application?.tag ?? null,
|
||||
},
|
||||
download: r.download ?? 0,
|
||||
upload: ulMap[r.application?.id] ?? 0,
|
||||
flows: r.flows ?? 0,
|
||||
}));
|
||||
}
|
||||
|
||||
async function fetchDiscoveredDevices(interval = 1440, limit = 500, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/local_ip/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/local_ip/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return null;
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const ip = r.local_ip?.address ?? String(r.local_ip);
|
||||
ulMap[ip] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
const ip = r.local_ip?.address ?? String(r.local_ip ?? '');
|
||||
return {
|
||||
ip_address: ip,
|
||||
mac_address: r.local_mac ?? null,
|
||||
device_label: r.device_label ?? ip,
|
||||
device_type: r.device_type ?? null,
|
||||
os_label: r.os_label ?? null,
|
||||
manufacturer: r.manufacturer ?? null,
|
||||
download: r.download ?? 0,
|
||||
upload: ulMap[ip] ?? 0,
|
||||
flows: r.flows ?? 0,
|
||||
last_seen: r.last_seen_at?.date ?? null,
|
||||
};
|
||||
}).filter(d => d.ip_address);
|
||||
}
|
||||
|
||||
async function fetchDeviceApps(ipAddress, interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
const ipFilter = JSON.stringify([ipAddress]);
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/application/download', { filter_interval: interval, settings_limit: limit, filter_local_ips: ipFilter }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/application/upload', { filter_interval: interval, settings_limit: limit, filter_local_ips: ipFilter }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData || !Array.isArray(dlData)) return [];
|
||||
const ulMap = {};
|
||||
if (ulData && Array.isArray(ulData)) {
|
||||
for (const r of ulData) {
|
||||
const id = r.application?.id;
|
||||
if (id) ulMap[id] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => ({
|
||||
app_label: r.application?.label ?? 'Unknown',
|
||||
app_id: r.application?.id ?? null,
|
||||
download: r.download ?? 0,
|
||||
upload: ulMap[r.application?.id] ?? 0,
|
||||
flows: r.flows ?? 0,
|
||||
})).filter(a => a.download > 0 || a.upload > 0);
|
||||
}
|
||||
|
||||
async function fetchFlows(limit = 10000, agentUuid = null, siteUuid = null) {
|
||||
const [raw, sniRaw] = await Promise.all([
|
||||
netifyFetch('/data/flows', { settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/tls_sni/download', { filter_interval: 1440, settings_limit: 50 }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!raw || !Array.isArray(raw)) return null;
|
||||
const sniList = [];
|
||||
if (sniRaw && Array.isArray(sniRaw)) {
|
||||
for (const r of sniRaw) {
|
||||
const sni = typeof r.tls_sni === 'object' ? r.tls_sni?.label : r.tls_sni;
|
||||
if (sni && typeof sni === 'string' && sni.trim() !== '') sniList.push(sni.replace(/^\*\./, '').trim());
|
||||
}
|
||||
}
|
||||
return raw.map(r => {
|
||||
const port = r.remote_port ?? null;
|
||||
const portService = port ? (PORT_SERVICE_MAP[port] ?? `Port ${port}`) : null;
|
||||
const appLabel = r.application?.label || portService;
|
||||
const sniVal = typeof r.tls_sni === 'object' ? r.tls_sni?.label : r.tls_sni;
|
||||
const domain = sniVal || r.dns_hostname || r.hostname || null;
|
||||
return {
|
||||
flow_id: String(r.flow_id ?? ''),
|
||||
src_ip: r.local_ip?.address ?? null,
|
||||
src_mac: r.local_mac ?? null,
|
||||
dst_ip: r.remote_ip?.address ?? null,
|
||||
dst_port: port,
|
||||
protocol: r.ip_protocol?.label ?? null,
|
||||
app_label: appLabel,
|
||||
domain: domain,
|
||||
download: r.download ?? 0,
|
||||
upload: r.upload ?? 0,
|
||||
first_seen: r.first_seen_at?.date ?? null,
|
||||
last_seen: r.last_seen_at?.date ?? null,
|
||||
};
|
||||
}).filter(f => f.src_ip);
|
||||
}
|
||||
|
||||
async function fetchCyberThreats(agentUuid = null, siteUuid = null) {
|
||||
const ipRepData = await netifyFetch('/data/stats/top/remote_ip/download', { filter_interval: 1440, settings_limit: 50 }, agentUuid, siteUuid);
|
||||
if (!ipRepData || !Array.isArray(ipRepData)) return [];
|
||||
const SUSPICIOUS_PORTS = new Set([23, 4444, 1337, 6667, 31337, 12345, 54321, 4899, 5554, 9999]);
|
||||
const threats = [];
|
||||
for (const r of ipRepData) {
|
||||
const ip = r.remote_ip?.address ?? null;
|
||||
const port = r.remote_port ?? 0;
|
||||
if (ip && SUSPICIOUS_PORTS.has(port)) {
|
||||
threats.push({
|
||||
threat_type: `Suspicious Port ${port}`,
|
||||
severity: 'High',
|
||||
src_ip: null,
|
||||
dst_ip: ip,
|
||||
dst_port: port,
|
||||
protocol: r.ip_protocol?.label ?? null,
|
||||
description: `Suspicious outbound connection to ${ip}:${port}`,
|
||||
event_at: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
}
|
||||
return threats;
|
||||
}
|
||||
|
||||
async function fetchEvents(limit = 100, agentUuid = null, siteUuid = null) {
|
||||
const raw = await netifyFetch('/event/events', { settings_limit: limit }, agentUuid, siteUuid);
|
||||
if (!raw || !Array.isArray(raw)) return [];
|
||||
return raw.map(r => {
|
||||
let msg = r.label || '';
|
||||
if (r.description) {
|
||||
try {
|
||||
const descObj = JSON.parse(r.description);
|
||||
msg = descObj.default || r.label || '';
|
||||
if (descObj.tags) {
|
||||
for (const k in descObj.tags) {
|
||||
const tagVal = descObj.tags[k];
|
||||
const val = Array.isArray(tagVal) ? (tagVal[0] === 'Unknown' && tagVal[1] ? tagVal[1] : tagVal[0]) : tagVal;
|
||||
msg = msg.replace(`{{ ${k} }}`, val).replace(`{{${k}}}`, val);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
msg = r.description;
|
||||
}
|
||||
}
|
||||
let sevLabel = 'Info';
|
||||
if (r.severity >= 30) sevLabel = 'Critical';
|
||||
else if (r.severity >= 20) sevLabel = 'High';
|
||||
else if (r.severity >= 10) sevLabel = 'Warning';
|
||||
let srcIp = null;
|
||||
if (r.description) {
|
||||
try {
|
||||
const descObj = JSON.parse(r.description);
|
||||
srcIp = descObj.tags?.device_ip || descObj.tags?.ip || null;
|
||||
} catch {}
|
||||
}
|
||||
return {
|
||||
event_id: r.id || null,
|
||||
event_type: r.basename || 'unknown',
|
||||
severity: sevLabel,
|
||||
description: msg,
|
||||
category_label: r.category?.label || 'Intelligence',
|
||||
ip_address: srcIp,
|
||||
mac_address: r.additional?.device?.mac?.address || null,
|
||||
event_at: r.created_at?.date ? new Date(r.created_at.date) : new Date()
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchAgents,
|
||||
fetchBandwidthSummary,
|
||||
fetchTopApps,
|
||||
fetchDiscoveredDevices,
|
||||
fetchDeviceApps,
|
||||
fetchFlows,
|
||||
fetchCyberThreats,
|
||||
fetchEvents,
|
||||
BASE_URL,
|
||||
PORT_SERVICE_MAP,
|
||||
...telemetry,
|
||||
};
|
||||
@@ -0,0 +1,77 @@
|
||||
// proxy/netifyClientCore.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Core fetch and authentication helpers for Netify DPI API.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
|
||||
const axios = require('axios');
|
||||
|
||||
const BASE_URL = process.env.NETIFY_INFORMATICS_BASE_URL || 'https://informatics.netify.ai/api/v1';
|
||||
const JWT_TOKEN = process.env.NETIFY_TOKEN || process.env.NETIFY_JWT_TOKEN;
|
||||
const agentMap = {};
|
||||
|
||||
function getHeaders(siteUuid) {
|
||||
const token = process.env.NETIFY_API_KEY || JWT_TOKEN;
|
||||
const headers = { 'x-api-key': token, 'Accept': 'application/json' };
|
||||
if (siteUuid) headers['x-net-site'] = siteUuid;
|
||||
return headers;
|
||||
}
|
||||
|
||||
function fixDates(obj) {
|
||||
if (Array.isArray(obj)) {
|
||||
for (let i = 0; i < obj.length; i++) fixDates(obj[i]);
|
||||
} else if (obj !== null && typeof obj === 'object') {
|
||||
for (const key in obj) {
|
||||
if ((key === 'first_seen_at' || key === 'last_seen_at' || key.endsWith('_at')) && obj[key] && typeof obj[key].date === 'string') {
|
||||
let d = obj[key].date;
|
||||
if (d.includes(' ') && !d.endsWith('Z')) obj[key].date = d.replace(' ', 'T') + 'Z';
|
||||
else if (!d.endsWith('Z') && !d.includes('+') && d.includes('T')) obj[key].date = d + 'Z';
|
||||
} else if (typeof obj[key] === 'object') {
|
||||
fixDates(obj[key]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function netifyFetch(endpoint, params = {}, agentUuid = null, siteUuid = null) {
|
||||
if (agentUuid) {
|
||||
const agentId = agentMap[agentUuid];
|
||||
if (agentId) {
|
||||
params.filter_agents = `[${agentId}]`;
|
||||
} else {
|
||||
params.settings_agent = agentUuid;
|
||||
}
|
||||
}
|
||||
|
||||
const token = process.env.NETIFY_API_KEY || JWT_TOKEN;
|
||||
if (!token) {
|
||||
console.error(`[DpiClient] Missing DPI_API_KEY for endpoint ${endpoint}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const res = await axios.get(`${BASE_URL}${endpoint}`, {
|
||||
headers: getHeaders(siteUuid), params, timeout: 30000,
|
||||
});
|
||||
const json = res.data;
|
||||
|
||||
if (json?.status_code !== 0) {
|
||||
console.error(`[DpiClient] API Error ${json?.status_code} on ${endpoint}: ${json?.status_message || 'No message'}`);
|
||||
return null;
|
||||
}
|
||||
if (json && json.data) fixDates(json.data);
|
||||
return json?.data ?? null;
|
||||
} catch (err) {
|
||||
const s = err.response?.status;
|
||||
const msg = JSON.stringify(err.response?.data ?? err.message);
|
||||
console.error(`[DpiClient] ${s ?? 'ERR'} ${endpoint}: ${msg}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
netifyFetch,
|
||||
agentMap,
|
||||
BASE_URL
|
||||
};
|
||||
@@ -0,0 +1,235 @@
|
||||
// proxy/netifyTelemetry.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Supplementary Telemetry endpoints wrapper for BackOne Proxy Server
|
||||
// Split from netifyClient.js to strictly respect the 256-line file size limit.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const { netifyFetch } = require('./netifyClientCore');
|
||||
|
||||
async function fetchTopAppCategories(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/application_category/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/application_category/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return [];
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const key = r.application_category?.label ?? r.application_category;
|
||||
if (key) ulMap[key] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
const label = r.application_category?.label ?? String(r.application_category ?? 'Unknown');
|
||||
return {
|
||||
category_label : label,
|
||||
download : r.download ?? 0,
|
||||
upload : ulMap[label] ?? 0,
|
||||
flows : r.flows ?? 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchTlsVersions(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/tls_version/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/tls_version/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return [];
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const key = r.tls_version?.label ?? r.tls_version?.code ?? r.tls_version;
|
||||
if (key) ulMap[key] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
const label = r.tls_version?.label ?? r.tls_version?.code ?? String(r.tls_version ?? 'Unknown');
|
||||
return {
|
||||
tls_version : label,
|
||||
download : r.download ?? 0,
|
||||
upload : ulMap[label] ?? 0,
|
||||
flows : r.flows ?? 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchTlsCiphers(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/tls_cipher/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/tls_cipher/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return [];
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const key = r.tls_cipher?.label ?? r.tls_cipher?.code ?? r.tls_cipher;
|
||||
if (key) ulMap[key] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
const label = r.tls_cipher?.label ?? r.tls_cipher?.code ?? String(r.tls_cipher ?? 'Unknown');
|
||||
return {
|
||||
tls_cipher : label,
|
||||
download : r.download ?? 0,
|
||||
upload : ulMap[label] ?? 0,
|
||||
flows : r.flows ?? 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchTlsSecurity(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/tls_security/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/tls_security/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return [];
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const key = r.tls_security?.label ?? r.tls_security?.code ?? r.tls_security;
|
||||
if (key) ulMap[key] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
const label = r.tls_security?.label ?? r.tls_security?.code ?? String(r.tls_security ?? 'Unknown');
|
||||
return {
|
||||
tls_security : label,
|
||||
download : r.download ?? 0,
|
||||
upload : ulMap[label] ?? 0,
|
||||
flows : r.flows ?? 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchTopCountries(interval = 1440, limit = 100, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/country/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/country/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return [];
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const cc = r.country?.code || r.country?.label;
|
||||
if (cc) ulMap[cc] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
const cc = r.country?.code || r.country?.label;
|
||||
return {
|
||||
country_code: cc || 'Unknown',
|
||||
country_name: r.country?.label ?? '',
|
||||
download: r.download ?? 0,
|
||||
upload: ulMap[r.country?.code] ?? 0,
|
||||
flows: r.flows ?? 0,
|
||||
};
|
||||
}).filter(r => r.country_code);
|
||||
}
|
||||
|
||||
async function fetchTopProperty(fieldName, interval, limit, agentUuid, siteUuid) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch(`/data/stats/top/${fieldName}/download`, { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch(`/data/stats/top/${fieldName}/upload`, { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return [];
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const item = r[fieldName];
|
||||
const key = item?.hash ?? item?.name ?? item?.label ?? String(item ?? '');
|
||||
if (key) ulMap[key] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
const item = r[fieldName];
|
||||
const key = item?.hash ?? item?.name ?? item?.label ?? String(item || 'Unknown');
|
||||
const label = item?.label ?? key;
|
||||
return {
|
||||
key,
|
||||
label,
|
||||
download: r.download ?? 0,
|
||||
upload: ulMap[key] ?? ulMap[label] ?? 0,
|
||||
flows: r.flows ?? 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function mapProp(data, keyName) {
|
||||
return data.map(d => ({
|
||||
[keyName]: d.key,
|
||||
download: d.download,
|
||||
upload: d.upload,
|
||||
flows: d.flows,
|
||||
}));
|
||||
}
|
||||
|
||||
async function fetchDhcpFingerprints(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('dhcp_class', interval, limit, agentUuid, siteUuid), 'fingerprint');
|
||||
}
|
||||
|
||||
async function fetchHttpUserAgents(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('http_useragent', interval, limit, agentUuid, siteUuid), 'user_agent');
|
||||
}
|
||||
|
||||
async function fetchBittorrentHashes(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
const data = await fetchTopProperty('bittorrent_info_hash', interval, limit, agentUuid, siteUuid);
|
||||
return data.map(d => ({
|
||||
info_hash: d.key,
|
||||
label: d.label,
|
||||
download: d.download,
|
||||
upload: d.upload,
|
||||
flows: d.flows,
|
||||
}));
|
||||
}
|
||||
|
||||
async function fetchSniHostnames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('tls_sni', interval, limit, agentUuid, siteUuid), 'sni_hostname');
|
||||
}
|
||||
|
||||
async function fetchSslServerCn(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('ssl_server_cn', interval, limit, agentUuid, siteUuid), 'ssl_server_cn');
|
||||
}
|
||||
|
||||
async function fetchQuicHostnames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('quic_hostname', interval, limit, agentUuid, siteUuid), 'quic_hostname');
|
||||
}
|
||||
|
||||
async function fetchSshClients(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('ssh_client', interval, limit, agentUuid, siteUuid), 'ssh_client');
|
||||
}
|
||||
|
||||
async function fetchSshServers(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('ssh_server', interval, limit, agentUuid, siteUuid), 'ssh_server');
|
||||
}
|
||||
|
||||
async function fetchMdnsHostnames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('mdns_hostname', interval, limit, agentUuid, siteUuid), 'mdns_hostname');
|
||||
}
|
||||
|
||||
async function fetchTopProtocols(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('ip_protocol', interval, limit, agentUuid, siteUuid), 'protocol_label');
|
||||
}
|
||||
|
||||
async function fetchSslSubjectAltNames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('ssl_subject_alt_name', interval, limit, agentUuid, siteUuid), 'alt_name');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchTopAppCategories,
|
||||
fetchTlsVersions,
|
||||
fetchTlsCiphers,
|
||||
fetchTlsSecurity,
|
||||
fetchTopCountries,
|
||||
fetchDhcpFingerprints,
|
||||
fetchHttpUserAgents,
|
||||
fetchBittorrentHashes,
|
||||
fetchSniHostnames,
|
||||
fetchSslServerCn,
|
||||
fetchQuicHostnames,
|
||||
fetchSshClients,
|
||||
fetchSshServers,
|
||||
fetchMdnsHostnames,
|
||||
fetchTopProtocols,
|
||||
fetchSslSubjectAltNames,
|
||||
};
|
||||
Generated
+1312
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "backone-proxy",
|
||||
"version": "1.0.0",
|
||||
"description": "BackOne DPI Proxy Server - Fetches from DPI API, filters per agent_uuid, stores to MongoDB",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"start": "node index.js",
|
||||
"start:bun": "bun run index.js",
|
||||
"dev": "nodemon index.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"axios": "^1.6.2",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^16.3.1",
|
||||
"express": "^4.18.2",
|
||||
"mongoose": "^8.0.3",
|
||||
"node-cron": "^3.0.3"
|
||||
}
|
||||
}
|
||||
+194
@@ -0,0 +1,194 @@
|
||||
// proxy/routes.js
|
||||
// ─── REST API Routes for BackOne Proxy Server ───────────────────────────────
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const mongoose = require('mongoose');
|
||||
const scheduler = require('./scheduler');
|
||||
|
||||
/**
|
||||
* GET /health
|
||||
* Liveness check
|
||||
*/
|
||||
router.get('/health', (req, res) => {
|
||||
const dbState = mongoose.connection.readyState;
|
||||
const dbLabel = ['disconnected', 'connected', 'connecting', 'disconnecting'][dbState] || 'unknown';
|
||||
res.json({
|
||||
ok: dbState === 1,
|
||||
service: 'BackOne Proxy Server',
|
||||
db: dbLabel,
|
||||
mode: process.env.PROXY_COLLECT_MODE || 'all',
|
||||
time: new Date().toISOString(),
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /status
|
||||
* Full scheduler + DB status
|
||||
*/
|
||||
router.get('/status', (req, res) => {
|
||||
const dbState = mongoose.connection.readyState;
|
||||
res.json({
|
||||
ok: true,
|
||||
db_state: ['disconnected', 'connected', 'connecting', 'disconnecting'][dbState] || 'unknown',
|
||||
...scheduler.getStatus(),
|
||||
time: new Date().toISOString(),
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /collect/all
|
||||
* Manual trigger — collect ALL agents sekarang
|
||||
*/
|
||||
router.post('/collect/all', async (req, res) => {
|
||||
if (mongoose.connection.readyState !== 1) {
|
||||
return res.status(503).json({ ok: false, error: 'MongoDB not connected. Cannot collect.' });
|
||||
}
|
||||
const { collectAllAgents } = require('./collector');
|
||||
console.log('[Proxy API] Manual trigger: collect ALL agents');
|
||||
try {
|
||||
const result = await collectAllAgents();
|
||||
res.json({ ok: result.success, ...result });
|
||||
} catch (err) {
|
||||
res.status(500).json({ ok: false, error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /collect/:agentUuid
|
||||
* Manual trigger — collect ONE specific agent
|
||||
*/
|
||||
router.post('/collect/:agentUuid', async (req, res) => {
|
||||
if (mongoose.connection.readyState !== 1) {
|
||||
return res.status(503).json({ ok: false, error: 'MongoDB not connected. Cannot collect.' });
|
||||
}
|
||||
const { agentUuid } = req.params;
|
||||
const { collectSpecificAgent } = require('./collector');
|
||||
console.log(`[Proxy API] Manual trigger: collect agent ${agentUuid}`);
|
||||
try {
|
||||
const result = await collectSpecificAgent(agentUuid);
|
||||
res.json({ ok: result.success, ...result });
|
||||
} catch (err) {
|
||||
res.status(500).json({ ok: false, error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /collect/agents
|
||||
* Manual trigger — collect multiple specific agents
|
||||
* Body: { "uuids": ["uuid1", "uuid2", ...], "delay_ms": 5000 }
|
||||
*/
|
||||
router.post('/collect/agents', async (req, res) => {
|
||||
if (mongoose.connection.readyState !== 1) {
|
||||
return res.status(503).json({ ok: false, error: 'MongoDB not connected. Cannot collect.' });
|
||||
}
|
||||
const { uuids, delay_ms } = req.body;
|
||||
if (!uuids || !Array.isArray(uuids) || uuids.length === 0) {
|
||||
return res.status(400).json({ ok: false, error: 'Body must include "uuids" as a non-empty array of agent UUIDs.' });
|
||||
}
|
||||
const { collectSpecificAgents } = require('./collector');
|
||||
console.log(`[Proxy API] Manual trigger: collect agents ${uuids.join(', ')}`);
|
||||
try {
|
||||
const result = await collectSpecificAgents(uuids, delay_ms || 5000);
|
||||
res.json({ ok: result.success, ...result });
|
||||
} catch (err) {
|
||||
res.status(500).json({ ok: false, error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /agents
|
||||
* List all agent UUIDs yang sudah tersimpan di MongoDB
|
||||
*/
|
||||
router.get('/agents', async (req, res) => {
|
||||
try {
|
||||
if (mongoose.connection.readyState !== 1) {
|
||||
return res.status(503).json({ ok: false, error: 'MongoDB not connected.' });
|
||||
}
|
||||
const { Summary } = require('./models/Schemas');
|
||||
const agents = await Summary.distinct('agent_uuid');
|
||||
res.json({ ok: true, data: agents });
|
||||
} catch (err) {
|
||||
res.status(500).json({ ok: false, error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /domain-details
|
||||
* Fetch live IP/MAC details for a specific domain from MongoDB
|
||||
*/
|
||||
router.get('/domain-details', async (req, res) => {
|
||||
const { domain, agentUuid } = req.query;
|
||||
if (!domain) return res.status(400).json({ ok: false, error: 'domain is required' });
|
||||
|
||||
try {
|
||||
const { Flow, DeviceStat } = require('./models/Schemas');
|
||||
|
||||
const filter = { domain: domain };
|
||||
if (agentUuid) {
|
||||
filter.agent_uuid = agentUuid;
|
||||
}
|
||||
|
||||
const raw = await Flow.find(filter).sort({ timestamp: -1 }).limit(100).lean();
|
||||
if (!raw || !Array.isArray(raw)) {
|
||||
return res.json({ ok: true, data: [] });
|
||||
}
|
||||
|
||||
const results = [];
|
||||
const seen = new Set();
|
||||
|
||||
for (const r of raw) {
|
||||
const ip = r.src_ip;
|
||||
const mac = r.src_mac;
|
||||
if (!ip || !mac) continue;
|
||||
|
||||
const key = `${ip}-${mac}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
let deviceName = 'Unknown Device';
|
||||
try {
|
||||
const dev = await DeviceStat.findOne({ mac_address: mac }).sort({ timestamp: -1 });
|
||||
if (dev && dev.name && dev.name !== 'Unknown Device') {
|
||||
deviceName = dev.name;
|
||||
}
|
||||
} catch (e) {
|
||||
// ignore timeout errors
|
||||
}
|
||||
|
||||
results.push({
|
||||
ip,
|
||||
mac,
|
||||
deviceName,
|
||||
lastSeen: r.timestamp || r.last_seen || r.first_seen,
|
||||
});
|
||||
}
|
||||
|
||||
res.json({ ok: true, data: results });
|
||||
} catch (err) {
|
||||
res.status(500).json({ ok: false, error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /latest
|
||||
* Tampilkan data terbaru dari semua collection (untuk debug)
|
||||
*/
|
||||
router.get('/latest', async (req, res) => {
|
||||
try {
|
||||
const { Summary, AppStat, DeviceStat, Flow, Threat, Event, AppCategoryStat } = require('./models/Schemas');
|
||||
const [summary, apps, devices, flows, threats, events, categories] = await Promise.all([
|
||||
Summary.findOne().sort({ timestamp: -1 }).lean(),
|
||||
AppStat.find().sort({ timestamp: -1 }).limit(5).lean(),
|
||||
DeviceStat.find().sort({ timestamp: -1 }).limit(5).lean(),
|
||||
Flow.find().sort({ timestamp: -1 }).limit(5).lean(),
|
||||
Threat.find().sort({ timestamp: -1 }).limit(5).lean(),
|
||||
Event.find().sort({ timestamp: -1 }).limit(5).lean(),
|
||||
AppCategoryStat.find().sort({ timestamp: -1 }).limit(5).lean(),
|
||||
]);
|
||||
res.json({ ok: true, data: { summary, apps, devices, flows, threats, events, categories } });
|
||||
} catch (err) {
|
||||
res.status(500).json({ ok: false, error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,133 @@
|
||||
// proxy/scheduler.js
|
||||
// Cron scheduler for automatic data collection from DPI API
|
||||
// Runs every 5 minutes, collecting data for all agents or a specific agent.
|
||||
|
||||
const cron = require('node-cron');
|
||||
const { collectAllAgents, collectSpecificAgent, collectSpecificAgents, populateLookupApps } = require('./collector');
|
||||
|
||||
// Mode: 'all' = collect all agents, 'agent' = collect one specific agent, 'agents' = collect multiple agents
|
||||
const COLLECT_MODE = process.env.PROXY_COLLECT_MODE || 'all';
|
||||
const SPECIFIC_AGENT = process.env.PROXY_AGENT_UUID || null;
|
||||
const SPECIFIC_AGENTS = (process.env.PROXY_AGENT_UUIDS || '').split(',').map(s => s.trim()).filter(Boolean);
|
||||
const AGENT_DELAY_MS = parseInt(process.env.PROXY_AGENT_DELAY_MS || '5000');
|
||||
const CRON_SCHEDULE = process.env.PROXY_CRON_SCHEDULE || '*/5 * * * *';
|
||||
|
||||
// Capacity logging is an expensive full-scan aggregation. Run it at most once per
|
||||
// interval (default 24h) instead of every collection cycle to reduce CPU/DB load.
|
||||
const CAPACITY_LOG_INTERVAL_MS = parseInt(process.env.PROXY_CAPACITY_LOG_INTERVAL_MS || String(24 * 60 * 60 * 1000));
|
||||
|
||||
let isRunning = false;
|
||||
let lastRunAt = null;
|
||||
let lastRunResult = null;
|
||||
let runCount = 0;
|
||||
let lastCapacityLogAt = 0;
|
||||
let lastAppLookupRefresh = 0;
|
||||
|
||||
/**
|
||||
* Execute one collection cycle (called by cron and manual trigger).
|
||||
* Prevents concurrent runs with isRunning guard.
|
||||
*/
|
||||
async function runCollection() {
|
||||
if (isRunning) {
|
||||
console.log('[Scheduler] Skipping - previous run still in progress');
|
||||
return { skipped: true, reason: 'already_running' };
|
||||
}
|
||||
|
||||
isRunning = true;
|
||||
lastRunAt = new Date();
|
||||
runCount++;
|
||||
|
||||
try {
|
||||
let result;
|
||||
if (COLLECT_MODE === 'agent' && SPECIFIC_AGENT) {
|
||||
console.log(`[Scheduler] Run #${runCount} - Mode: SPECIFIC AGENT (${SPECIFIC_AGENT})`);
|
||||
result = await collectSpecificAgent(SPECIFIC_AGENT);
|
||||
} else if (COLLECT_MODE === 'agents' && SPECIFIC_AGENTS.length > 0) {
|
||||
console.log(`[Scheduler] Run #${runCount} - Mode: SPECIFIC AGENTS (${SPECIFIC_AGENTS.join(', ')})`);
|
||||
result = await collectSpecificAgents(SPECIFIC_AGENTS, AGENT_DELAY_MS);
|
||||
} else {
|
||||
console.log(`[Scheduler] Run #${runCount} - Mode: ALL AGENTS`);
|
||||
result = await collectAllAgents();
|
||||
}
|
||||
lastRunResult = { ...result, run_count: runCount };
|
||||
|
||||
// Log MongoDB database capacity usage (expensive full-scan aggregation).
|
||||
// Only run periodically (default: every 24h) to avoid high CPU/DB load each cycle.
|
||||
const now = Date.now();
|
||||
if (now - lastCapacityLogAt >= CAPACITY_LOG_INTERVAL_MS) {
|
||||
lastCapacityLogAt = now;
|
||||
const { logCapacityStats } = require('./db/capacityTracker');
|
||||
await logCapacityStats(`[PROXY] [MongoDB] Capacity Used after Run #${runCount}:`);
|
||||
}
|
||||
|
||||
// Refresh application lookup catalog daily (expensive: fetches 5000+ apps)
|
||||
if (now - (lastAppLookupRefresh || 0) >= CAPACITY_LOG_INTERVAL_MS) {
|
||||
lastAppLookupRefresh = now;
|
||||
await populateLookupApps('').catch(err =>
|
||||
console.error('[Scheduler] LookupApp refresh failed:', err.message)
|
||||
);
|
||||
}
|
||||
|
||||
return lastRunResult;
|
||||
} catch (err) {
|
||||
console.error('[Scheduler] Unhandled error during collection:', err.message);
|
||||
lastRunResult = { success: false, error: err.message, run_count: runCount };
|
||||
return lastRunResult;
|
||||
} finally {
|
||||
isRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the scheduler (cron job + immediate first run).
|
||||
*/
|
||||
function startScheduler() {
|
||||
console.log(`[Scheduler] Starting proxy data collector`);
|
||||
const modeLabel = COLLECT_MODE === 'agent'
|
||||
? `SPECIFIC AGENT (${SPECIFIC_AGENT})`
|
||||
: COLLECT_MODE === 'agents'
|
||||
? `SPECIFIC AGENTS (${SPECIFIC_AGENTS.join(', ')})`
|
||||
: 'ALL AGENTS';
|
||||
console.log(`[Scheduler] Mode : ${modeLabel}`);
|
||||
console.log(`[Scheduler] Schedule : ${CRON_SCHEDULE} (every 5 minutes by default)`);
|
||||
|
||||
// Validate cron expression
|
||||
if (!cron.validate(CRON_SCHEDULE)) {
|
||||
console.error(`[Scheduler] Invalid cron expression: "${CRON_SCHEDULE}". Using default.`);
|
||||
}
|
||||
|
||||
// Start recurring cron job
|
||||
cron.schedule(CRON_SCHEDULE, () => {
|
||||
runCollection().catch(err => console.error('[Scheduler] Cron error:', err.message));
|
||||
});
|
||||
|
||||
console.log('[Scheduler] Cron job registered. Starting initial collection...');
|
||||
|
||||
// Initial run immediately on startup (async, do not block server start)
|
||||
setTimeout(async () => {
|
||||
// 1. Sync application lookup catalog from DPI API
|
||||
populateLookupApps('').catch(err => console.error('[Scheduler] Initial LookupApp sync failed:', err.message));
|
||||
|
||||
// 2. Start normal telemetry collection
|
||||
runCollection().catch(err => console.error('[Scheduler] Initial run error:', err.message));
|
||||
}, 2000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current scheduler status (for REST API endpoint).
|
||||
*/
|
||||
function getStatus() {
|
||||
return {
|
||||
is_running: isRunning,
|
||||
run_count: runCount,
|
||||
last_run_at: lastRunAt?.toISOString() ?? null,
|
||||
collect_mode: COLLECT_MODE,
|
||||
agent_uuid: SPECIFIC_AGENT,
|
||||
agent_uuids: COLLECT_MODE === 'agents' ? SPECIFIC_AGENTS : [],
|
||||
agent_delay_ms: AGENT_DELAY_MS,
|
||||
cron_schedule: CRON_SCHEDULE,
|
||||
last_result: lastRunResult,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { startScheduler, runCollection, getStatus };
|
||||
Reference in new issue
Block a user