feat(source2): push all latest files - device labeling, help system, proxy docs, database isolation fix
- Added DOKUMENTASI-FILTER-PER-SITE.md (site isolation docs) - Fixed start-with-env.js to force-load .env.production - Fixed MONGODB_URI hostname from mongodb-netify to mongodb.prod.proit.id - Updated .gitignore to exclude sensitive scripts and credential files - Minor UI and labeling improvements
This commit is contained in:
rafif committed
1 parent
a403f752f3
commit
dd4c8f6876
385 files changed
+31016
-8268
No files matched your search
+16
@@ -66,3 +66,19 @@ CLAUDE.md
|
||||
docs/
|
||||
plans/
|
||||
.env*
|
||||
|
||||
# Local uploads
|
||||
backend/public/api/uploads/
|
||||
|
||||
# Sensitive helper scripts (contain hardcoded SSH/API credentials - local use only)
|
||||
compare-netify-vs-dashboard.js
|
||||
ssh-read-source1-proxy.js
|
||||
check-frontend-uri-now.js
|
||||
verify-final.js
|
||||
check-frontend-uri.js
|
||||
ssh-check-logs.js
|
||||
ssh-*.js
|
||||
|
||||
# Sensitive documentation (contains production API keys / credentials)
|
||||
BUKTI-AKSES-MONGODB.txt
|
||||
DOKUMENTASI-PROXY-NETIFY.md
|
||||
@@ -0,0 +1,252 @@
|
||||
# DETAIL TEKNIS: Cara Proxy Memfilter Data per Site (SIAB vs Office)
|
||||
|
||||
Dokumen ini menjelaskan **secara kode** bagaimana data dipisahkan per site.
|
||||
Ada **3 lapis filter** yang bekerja dari Netify API sampai ke tampilan dashboard.
|
||||
|
||||
---
|
||||
|
||||
## LAPIS 1 — Saat Minta Data ke Netify API
|
||||
### File: `proxy/netifyClientCore.js`
|
||||
|
||||
```
|
||||
NETIFY_SITE_UUIDS = "6681452d_....(SIAB), 1959bb55_....(Office)"
|
||||
|
|
||||
proxy loop satu per satu:
|
||||
┌─────────────────────────┐
|
||||
│ for SIAB UUID: │
|
||||
│ kirim request ke │
|
||||
│ Netify dengan header │
|
||||
│ x-net-site: SIAB-UUID│
|
||||
└─────────────────────────┘
|
||||
┌─────────────────────────┐
|
||||
│ for Office UUID: │
|
||||
│ kirim request ke │
|
||||
│ Netify dengan header │
|
||||
│ x-net-site: OFFICE-UUID│
|
||||
└─────────────────────────┘
|
||||
```
|
||||
|
||||
**KODE ASLI — cara header dikirim:**
|
||||
```javascript
|
||||
// proxy/netifyClientCore.js baris 14-18
|
||||
function getHeaders(siteUuid) {
|
||||
const headers = {
|
||||
'x-api-key': process.env.NETIFY_API_KEY,
|
||||
'Accept': 'application/json'
|
||||
};
|
||||
|
||||
if (siteUuid) headers['x-net-site'] = siteUuid;
|
||||
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
// Ini yang memfilter data di sisi Netify!
|
||||
// Netify API hanya kembalikan data untuk site ini saja.
|
||||
|
||||
return headers;
|
||||
}
|
||||
|
||||
async function netifyFetch(endpoint, params = {}, agentUuid, siteUuid) {
|
||||
const res = await axios.get(`${BASE_URL}${endpoint}`, {
|
||||
headers: getHeaders(siteUuid), // <--- siteUuid dikirim ke Netify
|
||||
params,
|
||||
timeout: 30000,
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
**Artinya:** Netify API sendiri yang memfilter. Kalau kita kirim header
|
||||
`x-net-site: SIAB-UUID`, Netify HANYA kembalikan data milik SIAB.
|
||||
Kita tidak perlu filter manual — Netify sudah filter dari sumbernya.
|
||||
|
||||
---
|
||||
|
||||
## LAPIS 2 — Saat Simpan ke MongoDB
|
||||
### File: `proxy/collector.js` (loop utama)
|
||||
|
||||
Setelah data dari Netify masuk, setiap dokumen diberi **stempel `site_uuid`**
|
||||
sebelum disimpan ke MongoDB.
|
||||
|
||||
**KODE ASLI — loop per site di collector.js:**
|
||||
```javascript
|
||||
// proxy/collector.js baris 123-222
|
||||
|
||||
// SITE_UUIDS diambil dari env:
|
||||
// NETIFY_SITE_UUIDS="6681452d_..., 1959bb55_..."
|
||||
const SITE_UUIDS = SITE_UUIDS_STR.split(','); // ["SIAB-UUID", "OFFICE-UUID"]
|
||||
|
||||
for (const siteUuid of SITE_UUIDS) {
|
||||
// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
// Loop: pertama SIAB, lalu Office (satu per satu)
|
||||
|
||||
console.log(`Fetching agents for Site: ${siteUuid}`);
|
||||
const agents = await netify.fetchAgents(siteUuid);
|
||||
// ^^^^^^^^^
|
||||
// fetchAgents pakai siteUuid → Netify hanya beri agent milik site ini
|
||||
|
||||
// --- PENTING: Anti-duplikat antar site ---
|
||||
// Kadang Netify bisa kembalikan agent yang sama untuk 2 site.
|
||||
// Di sini kita cegah agar 1 agent hanya masuk 1 site.
|
||||
const agents = rawAgents.filter(a => {
|
||||
if (processedAgentUuids.has(a.uuid)) {
|
||||
console.log(`Skipping ${a.uuid} — already assigned to another site.`);
|
||||
return false; // lewati agent yang sudah diproses site lain
|
||||
}
|
||||
return true;
|
||||
});
|
||||
for (const agent of agents) processedAgentUuids.add(agent.uuid);
|
||||
|
||||
// Simpan agent ke MongoDB dengan site_uuid
|
||||
await AgentRegistry.findOneAndUpdate(
|
||||
{ uuid: agent.uuid },
|
||||
{ $set: {
|
||||
uuid: agent.uuid,
|
||||
site_uuid: siteUuid, // <--- stempel site di sini!
|
||||
...
|
||||
}},
|
||||
{ upsert: true }
|
||||
);
|
||||
|
||||
// Kumpulkan data untuk setiap agent di site ini
|
||||
for (const agent of agents) {
|
||||
await collectForAgent(agent.uuid, timestamp, siteUuid);
|
||||
// ^^^^^^^^^
|
||||
// siteUuid terus dibawa ke setiap fungsi collect
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**KODE ASLI — cara flows disimpan dengan site_uuid:**
|
||||
```javascript
|
||||
// proxy/collectorHelperDpi2.js baris 88-98
|
||||
|
||||
const flowDocs = flows.map(f => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid, // siapa agent-nya
|
||||
site_uuid: SITE_UUID, // <--- data ini milik site mana! (SIAB atau Office)
|
||||
flow_id: f.flow_id,
|
||||
src_ip: f.src_ip,
|
||||
dst_ip: f.dst_ip,
|
||||
download: f.download,
|
||||
upload: f.upload,
|
||||
// ...
|
||||
}));
|
||||
|
||||
// Upsert ke MongoDB (tidak duplikat berdasarkan flow_id + agent_uuid)
|
||||
await Flow.bulkWrite(flowDocs.map(f => ({
|
||||
updateOne: {
|
||||
filter: { flow_id: f.flow_id, agent_uuid: f.agent_uuid },
|
||||
update: { $set: f },
|
||||
upsert: true,
|
||||
}
|
||||
})));
|
||||
```
|
||||
|
||||
**Hasilnya di MongoDB — data terpisah per site:**
|
||||
```
|
||||
Collection: flows
|
||||
┌────────────────────┬────────────────────────────────────────────────────┬────────┬──────────┐
|
||||
│ flow_id │ site_uuid │ src_ip │ download │
|
||||
├────────────────────┼────────────────────────────────────────────────────┼────────┼──────────┤
|
||||
│ flow-001 │ 6681452d_9cae_4ff4_8ae8_0d504774265e (SIAB) │ 10.0.x │ 1234 │
|
||||
│ flow-002 │ 6681452d_9cae_4ff4_8ae8_0d504774265e (SIAB) │ 10.0.x │ 5678 │
|
||||
│ flow-003 │ 1959bb55_045b_47c7_bbdd_f33b7db197b9 (Office) │ 192.168.x │ 9012 │
|
||||
│ flow-004 │ 1959bb55_045b_47c7_bbdd_f33b7db197b9 (Office) │ 192.168.x │ 3456 │
|
||||
└────────────────────┴────────────────────────────────────────────────────┴────────┴──────────┘
|
||||
^^^^^^^^^^ Field ini yang memisahkan data ^^^^^^^^^^
|
||||
```
|
||||
|
||||
**Semua collection lain juga sama:**
|
||||
- `devices` → tiap dokumen ada `site_uuid`
|
||||
- `threats` → tiap dokumen ada `site_uuid`
|
||||
- `events` → tiap dokumen ada `site_uuid`
|
||||
- `summaries` → tiap dokumen ada `site_uuid`
|
||||
- `telemetry` → tiap dokumen ada `site_uuid`
|
||||
|
||||
---
|
||||
|
||||
## LAPIS 3 — Saat Dashboard Baca dari MongoDB
|
||||
### File: `backend/routes/dashboard/flows.js` (contoh)
|
||||
|
||||
Ketika user login sebagai admin SIAB dan buka halaman Flows,
|
||||
backend hanya query dokumen dengan `site_uuid` yang sesuai:
|
||||
|
||||
```javascript
|
||||
// backend/routes/dashboard/flows.js (contoh query)
|
||||
const userSiteUuid = req.user.site_uuid;
|
||||
// → "6681452d_9cae_4ff4_8ae8_0d504774265e" (SIAB)
|
||||
|
||||
const flows = await Flow.find({
|
||||
site_uuid: userSiteUuid, // <--- hanya ambil data site ini!
|
||||
// ...filter waktu, pagination, dsb
|
||||
}).limit(50);
|
||||
```
|
||||
|
||||
Admin Office login → `site_uuid = 1959bb55_...` → hanya lihat data Office.
|
||||
Admin SIAB login → `site_uuid = 6681452d_...` → hanya lihat data SIAB.
|
||||
Super Admin → bisa pilih site mana yang ingin dilihat.
|
||||
|
||||
---
|
||||
|
||||
## RINGKASAN — Alur Lengkap Filter Data
|
||||
|
||||
```
|
||||
Netify API
|
||||
|
|
||||
|-- Lapis 1: Header x-net-site dikirim ke Netify
|
||||
| Netify hanya kirim data milik site tersebut
|
||||
|
|
||||
v
|
||||
Proxy Server (setiap 5 menit)
|
||||
|
|
||||
|-- Lapis 2: Setiap dokumen diberi stempel site_uuid
|
||||
| - SIAB data → { site_uuid: "6681452d_..." }
|
||||
| - Office data → { site_uuid: "1959bb55_..." }
|
||||
| - Anti-duplikat: 1 agent hanya masuk 1 site
|
||||
|
|
||||
v
|
||||
MongoDB (semua data tercampur tapi ter-tag per site)
|
||||
|
|
||||
|-- Lapis 3: Backend query MongoDB dengan filter site_uuid
|
||||
| - Admin SIAB login → WHERE site_uuid = SIAB-UUID
|
||||
| - Admin Office login → WHERE site_uuid = OFFICE-UUID
|
||||
|
|
||||
v
|
||||
Web Dashboard (tampil hanya data site yang sesuai)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Skenario Konkret
|
||||
|
||||
**Skenario:** Network agent "F6-2V-DT-8A" ada di SIAB. Network agent "23-TE-6L-I2" ada di Office.
|
||||
|
||||
### Langkah 1 — Proxy request ke Netify
|
||||
```
|
||||
[Iter 1] siteUuid = "6681452d..." (SIAB)
|
||||
→ GET /data/flows
|
||||
Header: x-net-site: 6681452d...
|
||||
→ Netify kembalikan: flows dari F6-2V-DT-8A (agent SIAB)
|
||||
→ Simpan ke MongoDB: { site_uuid: "6681452d...", agent_uuid: "F6-2V-DT-8A", flow_id: ... }
|
||||
|
||||
[Iter 2] siteUuid = "1959bb55..." (Office)
|
||||
→ GET /data/flows
|
||||
Header: x-net-site: 1959bb55...
|
||||
→ Netify kembalikan: flows dari 23-TE-6L-I2 (agent Office)
|
||||
→ Simpan ke MongoDB: { site_uuid: "1959bb55...", agent_uuid: "23-TE-6L-I2", flow_id: ... }
|
||||
```
|
||||
|
||||
### Langkah 2 — Dashboard tampilkan
|
||||
```
|
||||
User siab login:
|
||||
req.user.site_uuid = "6681452d..."
|
||||
DB query: Flow.find({ site_uuid: "6681452d..." })
|
||||
Hasil: hanya flow dari F6-2V-DT-8A ✓
|
||||
|
||||
User office login:
|
||||
req.user.site_uuid = "1959bb55..."
|
||||
DB query: Flow.find({ site_uuid: "1959bb55..." })
|
||||
Hasil: hanya flow dari 23-TE-6L-I2 ✓
|
||||
```
|
||||
|
||||
**Data tidak pernah tercampur** karena ada 3 lapis isolasi ini.
|
||||
|
||||
---
|
||||
*Dokumentasi teknis Source 2 — 29 Juli 2026*
|
||||
@@ -41,7 +41,7 @@ Produk BackOne oleh **PT. Data Bisnis Solusi** — Dashboard monitoring jaringan
|
||||
|
||||
## 📡 Proxy — 2 Mode Pengambilan Data
|
||||
|
||||
Proxy server (port 4000) mendukung 3 mode yang dikontrol via environment variable:
|
||||
Proxy server (port 4000) mendukung 2 mode yang dikontrol via environment variable:
|
||||
|
||||
### Mode 1: Semua Network Agent (Admin BackOne)
|
||||
|
||||
@@ -62,26 +62,14 @@ PROXY_AGENT_UUID=2F-TF-1D-GK # UUID Network Agent CPI Balaraja
|
||||
|
||||
Proxy hanya mengambil data dari **satu Network Agent spesifik** (berdasarkan UUID). Data agent lain tidak pernah masuk ke database. Cocok untuk deployment di sisi client (Pihak A, B, C) agar mereka hanya punya data milik mereka sendiri.
|
||||
|
||||
### Mode 3: Beberapa Agent Spesifik (Multi-Agent)
|
||||
|
||||
```env
|
||||
# .env.local
|
||||
PROXY_COLLECT_MODE=agents
|
||||
PROXY_AGENT_UUIDS=UUID-AGENT-A,UUID-AGENT-B,UUID-AGENT-C # comma-separated list
|
||||
PROXY_AGENT_DELAY_MS=5000 # delay antar agent (default: 5000ms)
|
||||
```
|
||||
|
||||
Proxy mengambil data dari **beberapa Network Agent spesifik** (berdasarkan daftar UUID yang dipisahkan koma). Data agent di luar daftar tidak pernah masuk ke database. Delay antar agent dapat diatur dengan `PROXY_AGENT_DELAY_MS` untuk menghindari rate-limit.
|
||||
|
||||
### Contoh Multi-Tenant Deployment
|
||||
|
||||
| Deployment | PROXY_COLLECT_MODE | PROXY_AGENT_UUID / PROXY_AGENT_UUIDS | Data yang disimpan |
|
||||
| Deployment | PROXY_COLLECT_MODE | PROXY_AGENT_UUID | Data yang disimpan |
|
||||
|---|---|---|---|
|
||||
| Kantor BackOne (Admin) | `all` | _(kosong)_ | Semua agent |
|
||||
| Pihak A | `agent` | `UUID-AGENT-A` | Hanya data Pihak A |
|
||||
| Pihak B | `agent` | `UUID-AGENT-B` | Hanya data Pihak B |
|
||||
| Pihak C | `agent` | `UUID-AGENT-C` | Hanya data Pihak C |
|
||||
| Multi-Client | `agents` | `UUID-A,UUID-B` | Data Pihak A dan B |
|
||||
|
||||
---
|
||||
|
||||
@@ -94,7 +82,6 @@ Proxy mengambil data dari **beberapa Network Agent spesifik** (berdasarkan dafta
|
||||
| GET | `/agents` | List semua agent UUID yang ada di MongoDB |
|
||||
| POST | `/collect/all` | Trigger manual — kumpulkan semua agent |
|
||||
| POST | `/collect/:uuid` | Trigger manual — kumpulkan agent spesifik |
|
||||
| POST | `/collect/agents` | Trigger manual — kumpulkan multiple agents (body: `{"uuids": [...], "delay_ms": 5000}`) |
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -26,9 +26,7 @@ contract, not just a task description.
|
||||
|
||||
**Responsibilities**
|
||||
- Implement API/data-layer logic.
|
||||
- Wire the mock-vs-live routing required by the Demo/Live switch (`AGENTS.md` §5) and
|
||||
the Cloud/Local endpoint switch (`AGENTS.md` §6) — both must resolve through the
|
||||
same contract so swapping either setting never changes calling code.
|
||||
- Ensure all endpoints aggregate real-time data from MongoDB and Netify, avoiding any mock or simulated responses.
|
||||
- Keep business logic out of route handlers; route handlers stay thin.
|
||||
|
||||
**When invoked**: any task touching data, APIs, or service integration.
|
||||
@@ -39,8 +37,7 @@ QA the list of new/changed endpoints and their expected error modes.
|
||||
## 3. Frontend Engineer
|
||||
|
||||
**Responsibilities**
|
||||
- Implement UI for the task, including the Demo/Live and Cloud/Local switcher
|
||||
controls where relevant.
|
||||
- Implement UI for the task.
|
||||
- Consume the Backend's contract rather than reaching around it.
|
||||
- Keep components small and composable, respecting the 256-LOC rule.
|
||||
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
FROM oven/bun:1-alpine
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY backend/package*.json ./
|
||||
RUN bun install --production
|
||||
|
||||
COPY backend/ .
|
||||
|
||||
EXPOSE 3001
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=20s --retries=3 \
|
||||
CMD bun -e "require('http').get('http://localhost:3001/api/health', r => r.statusCode === 200 ? process.exit(0) : process.exit(1)).on('error', () => process.exit(1))"
|
||||
|
||||
CMD ["bun", "run", "server.js"]
|
||||
@@ -0,0 +1,3 @@
|
||||
const db = require('better-sqlite3')('backend/netify_data.db');
|
||||
console.log('Devices:', db.prepare("SELECT * FROM devices WHERE ip_address = '192.168.9.2'").all());
|
||||
console.log('Discovery:', db.prepare("SELECT * FROM intel_device_discovery WHERE ip_address = '192.168.9.2'").all());
|
||||
@@ -0,0 +1,22 @@
|
||||
const mongoose = require('mongoose');
|
||||
require('dotenv').config({path: '../.env.local'});
|
||||
mongoose.connect(process.env.MONGODB_URI).then(async () => {
|
||||
const db = mongoose.connection;
|
||||
const highEvents = await db.collection('events').find({
|
||||
$or: [
|
||||
{severity: {$in: ['Critical', 'High']}},
|
||||
{category_label: 'Cybersecurity'}
|
||||
]
|
||||
}).toArray();
|
||||
|
||||
if (highEvents.length > 0) {
|
||||
console.log("High Events timestamps:");
|
||||
highEvents.forEach(e => {
|
||||
console.log("- event_at:", e.event_at, " | timestamp:", e.timestamp);
|
||||
});
|
||||
} else {
|
||||
console.log("No high events found in array");
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
}).catch(e => console.error(e));
|
||||
@@ -0,0 +1,44 @@
|
||||
const mongoose = require('mongoose');
|
||||
|
||||
mongoose.connect('mongodb://backone_user:SusuKudaLiar@103.80.237.29:27017/backone_dpi?authSource=backone_dpi')
|
||||
.then(async () => {
|
||||
const db = mongoose.connection.useDb('backone_dpi');
|
||||
const yesterday = new Date(Date.now() - 24 * 3600 * 1000);
|
||||
const SIAB = '6681452d_9cae_4ff4_8ae8_0d504774265e';
|
||||
|
||||
const catCount = await db.db.collection('appcategorystats').countDocuments({ site_uuid: SIAB, timestamp: { $gte: yesterday } });
|
||||
const catSum = await db.db.collection('appcategorystats').aggregate([
|
||||
{ $match: { site_uuid: SIAB, timestamp: { $gte: yesterday } } },
|
||||
{ $group: { _id: null, dl: { $sum: '$download' }, ul: { $sum: '$upload' } } }
|
||||
]).toArray();
|
||||
|
||||
const sumCount = await db.db.collection('summaries').countDocuments({ site_uuid: SIAB, timestamp: { $gte: yesterday } });
|
||||
const sumSum = await db.db.collection('summaries').aggregate([
|
||||
{ $match: { site_uuid: SIAB, timestamp: { $gte: yesterday } } },
|
||||
{ $group: { _id: null, dl: { $sum: '$bandwidth_down' }, ul: { $sum: '$bandwidth_up' } } }
|
||||
]).toArray();
|
||||
|
||||
const flowCount = await db.db.collection('flows').countDocuments({ site_uuid: SIAB, timestamp: { $gte: yesterday } });
|
||||
const flowSum = await db.db.collection('flows').aggregate([
|
||||
{ $match: { site_uuid: SIAB, timestamp: { $gte: yesterday } } },
|
||||
{ $group: { _id: null, dl: { $sum: '$download' }, ul: { $sum: '$upload' } } }
|
||||
]).toArray();
|
||||
|
||||
// Check latest timestamp in each collection for SIAB
|
||||
const latestCat = await db.db.collection('appcategorystats').findOne({ site_uuid: SIAB }, { sort: { timestamp: -1 } });
|
||||
const latestFlow = await db.db.collection('flows').findOne({ site_uuid: SIAB }, { sort: { timestamp: -1 } });
|
||||
const latestSum = await db.db.collection('summaries').findOne({ site_uuid: SIAB }, { sort: { timestamp: -1 } });
|
||||
|
||||
console.log('=== SIAB Site Data Check (Last 24h) ===');
|
||||
console.log('AppCatStats (24h):', catCount, 'docs | Sum:', JSON.stringify(catSum[0]));
|
||||
console.log('Summaries (24h) :', sumCount, 'docs | Sum:', JSON.stringify(sumSum[0]));
|
||||
console.log('Flows (24h) :', flowCount, 'docs | Sum:', JSON.stringify(flowSum[0]));
|
||||
console.log('');
|
||||
console.log('=== Latest Timestamps ===');
|
||||
console.log('Latest AppCat :', latestCat?.timestamp);
|
||||
console.log('Latest Flow :', latestFlow?.timestamp);
|
||||
console.log('Latest Summary :', latestSum?.timestamp);
|
||||
|
||||
mongoose.disconnect();
|
||||
})
|
||||
.catch(e => { console.error('Error:', e.message); process.exit(1); });
|
||||
@@ -0,0 +1,31 @@
|
||||
const { Client } = require('ssh2');
|
||||
const conn = new Client();
|
||||
|
||||
conn.on('ready', () => {
|
||||
const cmd = [
|
||||
'export PM2=/home/adminbackend/.npm-global/bin/pm2',
|
||||
'$PM2 list',
|
||||
'echo "=== MEMORY ==="',
|
||||
'free -m',
|
||||
'echo "=== DISK ==="',
|
||||
'df -h /',
|
||||
'echo "=== FRONTEND LOGS ==="',
|
||||
'$PM2 logs backone-frontend --lines 20 --nostream 2>&1',
|
||||
'echo "=== BACKEND LOGS ==="',
|
||||
'$PM2 logs backone-backend --lines 10 --nostream 2>&1',
|
||||
].join(' && ');
|
||||
|
||||
conn.exec(cmd, (err, stream) => {
|
||||
if (err) { console.error(err); conn.end(); return; }
|
||||
stream.on('data', d => process.stdout.write(d.toString()));
|
||||
stream.stderr.on('data', d => process.stderr.write(d.toString()));
|
||||
stream.on('close', () => conn.end());
|
||||
});
|
||||
}).connect({
|
||||
host: '103.185.47.52',
|
||||
port: 2222,
|
||||
username: 'adminbackend',
|
||||
password: 'htEo7x6LsBQiEHHH',
|
||||
});
|
||||
|
||||
conn.on('error', e => console.error('SSH Error:', e.message));
|
||||
@@ -0,0 +1,15 @@
|
||||
const mongoose = require('mongoose');
|
||||
require('dotenv').config({path: '../.env.local'});
|
||||
mongoose.connect(process.env.MONGODB_URI).then(async () => {
|
||||
const db = mongoose.connection;
|
||||
const threats = await db.collection('threats').countDocuments();
|
||||
const events = await db.collection('events').countDocuments();
|
||||
const highEvents = await db.collection('events').countDocuments({
|
||||
$or: [
|
||||
{severity: {$in: ['Critical', 'High']}},
|
||||
{category_label: 'Cybersecurity'}
|
||||
]
|
||||
});
|
||||
console.log({threats, events, highEvents});
|
||||
process.exit(0);
|
||||
}).catch(e => console.error(e));
|
||||
@@ -0,0 +1,10 @@
|
||||
const mongoose = require('mongoose');
|
||||
require('dotenv').config({path: '../.env.local'});
|
||||
mongoose.connect(process.env.MONGODB_URI).then(async () => {
|
||||
const db = mongoose.connection;
|
||||
const threats = await db.collection('threats').aggregate([{ $group: { _id: '$threat_type', count: { $sum: 1 } } }]).toArray();
|
||||
console.log('Threat types:', threats);
|
||||
const events = await db.collection('events').aggregate([{ $group: { _id: '$event_type', count: { $sum: 1 } } }]).toArray();
|
||||
console.log('Event types:', events);
|
||||
process.exit(0);
|
||||
});
|
||||
+2146
File diff suppressed because it is too large.
Load diff
@@ -5,7 +5,8 @@
|
||||
|
||||
const mongoose = require('mongoose');
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.join(__dirname, '../../.env.local') });
|
||||
const envFile = process.env.NODE_ENV === 'production' ? '.env.production' : '.env.local';
|
||||
require('dotenv').config({ path: path.join(__dirname, '../../', envFile) });
|
||||
|
||||
const MONGODB_URI = process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017/backone_dpi';
|
||||
|
||||
@@ -22,7 +23,7 @@ async function connectDB() {
|
||||
serverSelectionTimeoutMS: 10000,
|
||||
connectTimeoutMS: 10000,
|
||||
});
|
||||
console.log('[MongoDB] ✓ Connected successfully');
|
||||
console.log('[MongoDB] ✓ Connected successfully to', MONGODB_URI);
|
||||
const { logCapacityStats } = require('./capacityTracker');
|
||||
logCapacityStats('[MongoDB]').catch(err => console.warn('[MongoDB] Capacity log failed:', err.message));
|
||||
return;
|
||||
|
||||
+400
-400
@@ -1,400 +1,400 @@
|
||||
/**
|
||||
* generate_export.js
|
||||
*
|
||||
* Mengekspor SELURUH data dari semua tabel SQLite (database)
|
||||
* ke dalam file backone_data_export.txt
|
||||
*
|
||||
* Format output:
|
||||
* - Header metadata (tanggal, versi, jumlah tabel)
|
||||
* - Untuk setiap tabel: header section, row count, schema, dan semua data (JSON per baris)
|
||||
* - Footer summary
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const db = require('./database');
|
||||
|
||||
const OUTPUT_FILE = path.join(__dirname, '../backone_data_export.txt');
|
||||
const d = db.getDB();
|
||||
|
||||
// ─── Helpers ────────────────────────────────────────────────────────────────
|
||||
function fmtBytes(bytes) {
|
||||
if (!bytes || bytes === 0) return '0 B';
|
||||
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||
let b = Math.abs(bytes);
|
||||
let i = 0;
|
||||
while (b >= 1024 && i < units.length - 1) { b /= 1024; i++; }
|
||||
return b.toFixed(2) + ' ' + units[i];
|
||||
}
|
||||
|
||||
function fmtNum(n) {
|
||||
if (n == null) return 'N/A';
|
||||
return Number(n).toLocaleString('id-ID');
|
||||
}
|
||||
|
||||
function separator(char = '═', len = 80) {
|
||||
return char.repeat(len);
|
||||
}
|
||||
|
||||
function sectionHeader(tableName, rowCount, description) {
|
||||
return [
|
||||
'',
|
||||
separator('═'),
|
||||
`[TABLE: ${tableName}]`,
|
||||
`Row Count: ${fmtNum(rowCount)}`,
|
||||
description ? `Description: ${description}` : '',
|
||||
separator('─'),
|
||||
].filter(l => l !== '').join('\n');
|
||||
}
|
||||
|
||||
// ─── Table descriptions ──────────────────────────────────────────────────────
|
||||
const TABLE_DESCRIPTIONS = {
|
||||
bandwidth_apps : 'Bandwidth per aplikasi (YouTube, Facebook, dll) dari BackOne DPI',
|
||||
bandwidth_timeline : 'Timeline bandwidth per menit (download/upload historis)',
|
||||
bittorrent_info_hashes: 'Deteksi aktivitas BitTorrent berdasarkan info hash',
|
||||
countries : 'Distribusi traffic berdasarkan negara tujuan',
|
||||
devices : 'Daftar perangkat (IP/MAC) beserta bandwidth & OS',
|
||||
dhcp_fingerprints : 'Fingerprint DHCP untuk identifikasi tipe device',
|
||||
discovered_os : 'OS yang terdeteksi dari traffic scanning',
|
||||
dns_stats : 'Query DNS teratas dan statistik resolusi domain',
|
||||
events : 'Event log dari BackOne agent (koneksi, peringatan, dll)',
|
||||
flows : 'Data aliran jaringan per-sesi (src IP, dst IP, aplikasi, domain, bytes)',
|
||||
flow_origins : 'Asal flow: lokal (LAN) atau eksternal (WAN)',
|
||||
flow_types : 'Tipe flow: TCP, UDP, ICMP, dll',
|
||||
http_user_agents : 'HTTP User-Agent yang terdeteksi (browser, OS, framework)',
|
||||
intel_crypto_mining : 'Deteksi aktivitas crypto mining (pool host, protokol)',
|
||||
intel_device_discovery: 'Penemuan perangkat baru di jaringan (tipe, OS, manufaktur)',
|
||||
intel_encryption_audit: 'Audit enkripsi traffic per perangkat (encrypted%, risk level)',
|
||||
intel_insecure_protocols: 'Protokol tidak aman yang terdeteksi (HTTP, Telnet, FTP, dll)',
|
||||
intel_ip_reputation : 'Reputasi IP eksternal (blacklist, threat score)',
|
||||
intel_server_discovery: 'Server yang terdeteksi (HTTPS, SSH, HTTP, dll)',
|
||||
intel_tor_detection : 'Deteksi penggunaan jaringan Tor',
|
||||
intel_unencrypted_passwords: 'Deteksi pengiriman password dalam bentuk plaintext',
|
||||
intel_vpn_detection : 'Deteksi penggunaan VPN (OpenVPN, WireGuard, dll)',
|
||||
interfaces : 'Interface jaringan per agent (WAN/LAN, bandwidth)',
|
||||
ip_versions : 'Distribusi traffic IPv4 vs IPv6',
|
||||
mac_bandwidth : 'Bandwidth per MAC address perangkat',
|
||||
mdns_hostnames : 'mDNS hostname yang terdeteksi di jaringan lokal',
|
||||
netbios_hostnames : 'NetBIOS hostname (nama komputer Windows)',
|
||||
protocols : 'Distribusi protokol jaringan (port usage)',
|
||||
quic_hostnames : 'Hostname via QUIC/HTTP3 (Google, Cloudflare, dll)',
|
||||
regions : 'Distribusi traffic berdasarkan region/kota tujuan',
|
||||
remote_ips : 'IP remote teratas yang diakses perangkat',
|
||||
sni_hostnames : 'Server Name Indication dari koneksi TLS',
|
||||
ssh_versions : 'Versi SSH yang terdeteksi di jaringan',
|
||||
ssl_server_cn : 'Common Name sertifikat SSL server',
|
||||
threats : 'Ancaman keamanan terdeteksi (threat alerts)',
|
||||
tls_ciphers : 'Cipher suite TLS yang digunakan',
|
||||
tls_security : 'Tingkat keamanan TLS (Modern, Compatible, Old)',
|
||||
tls_versions : 'Versi TLS yang digunakan (1.0, 1.2, 1.3)',
|
||||
vlans : 'VLAN yang terdeteksi di jaringan',
|
||||
};
|
||||
|
||||
// ─── Main Export Logic ───────────────────────────────────────────────────────
|
||||
async function main() {
|
||||
console.log('🚀 Memulai export data...');
|
||||
|
||||
const exportDate = new Date().toISOString();
|
||||
const lines = [];
|
||||
|
||||
// ── File Header ──────────────────────────────────────────────────────────
|
||||
lines.push(separator('═'));
|
||||
lines.push(' BACKONE DATA EXPORT');
|
||||
lines.push(' Seluruh data hasil parsing dari BackOne API');
|
||||
lines.push(separator('─'));
|
||||
lines.push(` Export Date: ${exportDate}`);
|
||||
lines.push(` Generated by: generate_export.js`);
|
||||
lines.push(` Source: database (SQLite lokal)`);
|
||||
lines.push(` API Base: BackOne API Service`);
|
||||
lines.push(` Format: Per-tabel, data JSON satu record per baris (JSONL)`);
|
||||
lines.push(separator('─'));
|
||||
|
||||
// ── Get all tables ────────────────────────────────────────────────────────
|
||||
const tables = d.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
|
||||
).all().map(r => r.name);
|
||||
|
||||
lines.push(` Total Tables: ${tables.length}`);
|
||||
lines.push(separator('═'));
|
||||
lines.push('');
|
||||
|
||||
// ── Table of Contents ─────────────────────────────────────────────────────
|
||||
lines.push('TABLE OF CONTENTS');
|
||||
lines.push(separator('─', 40));
|
||||
let totalRows = 0;
|
||||
const tableSummaries = [];
|
||||
for (const tableName of tables) {
|
||||
const cnt = d.prepare(`SELECT COUNT(*) as c FROM ${tableName}`).get().c;
|
||||
totalRows += cnt;
|
||||
const desc = TABLE_DESCRIPTIONS[tableName] || '-';
|
||||
lines.push(` ${tableName.padEnd(35)} ${String(cnt).padStart(8)} rows`);
|
||||
tableSummaries.push({ name: tableName, count: cnt, description: desc });
|
||||
}
|
||||
lines.push(separator('─', 40));
|
||||
lines.push(` ${'TOTAL'.padEnd(35)} ${String(totalRows).padStart(8)} rows`);
|
||||
lines.push('');
|
||||
|
||||
// ── Per-Table Export ──────────────────────────────────────────────────────
|
||||
for (const { name: tableName, count, description } of tableSummaries) {
|
||||
console.log(` 📋 Exporting: ${tableName} (${fmtNum(count)} rows)...`);
|
||||
|
||||
// Section header
|
||||
lines.push(sectionHeader(tableName, count, description));
|
||||
|
||||
// Schema
|
||||
const cols = d.prepare(`PRAGMA table_info(${tableName})`).all();
|
||||
lines.push('Schema:');
|
||||
cols.forEach(c => {
|
||||
lines.push(` - ${c.name} [${c.type || 'TEXT'}]${c.notnull ? ' NOT NULL' : ''}${c.pk ? ' PRIMARY KEY' : ''}`);
|
||||
});
|
||||
lines.push('');
|
||||
|
||||
// Statistics for numeric columns
|
||||
const numericCols = cols.filter(c =>
|
||||
['INTEGER', 'REAL', 'NUMERIC'].includes((c.type || '').toUpperCase()) &&
|
||||
!['id'].includes(c.name.toLowerCase())
|
||||
);
|
||||
|
||||
if (count > 0 && numericCols.length > 0) {
|
||||
lines.push('Statistics:');
|
||||
for (const col of numericCols.slice(0, 5)) { // max 5 numeric cols
|
||||
try {
|
||||
const stat = d.prepare(`
|
||||
SELECT MIN(${col.name}) as min, MAX(${col.name}) as max,
|
||||
AVG(${col.name}) as avg, SUM(${col.name}) as total
|
||||
FROM ${tableName}
|
||||
`).get();
|
||||
if (stat && stat.max !== null) {
|
||||
lines.push(` ${col.name}: min=${fmtNum(stat.min)} max=${fmtNum(stat.max)} avg=${Number(stat.avg || 0).toFixed(2)} total=${fmtNum(stat.total)}`);
|
||||
}
|
||||
} catch(e) { /* skip */ }
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Data rows (ALL rows)
|
||||
if (count === 0) {
|
||||
lines.push('(No data)');
|
||||
} else {
|
||||
lines.push(`Data (${fmtNum(count)} records):`);
|
||||
const rows = d.prepare(`SELECT * FROM ${tableName}`).all();
|
||||
for (const row of rows) {
|
||||
lines.push(JSON.stringify(row));
|
||||
}
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// ── Agent-specific sections (derived from flows) ───────────────────────────
|
||||
lines.push('');
|
||||
lines.push(separator('═'));
|
||||
lines.push('[DERIVED: AGENT ANALYSIS]');
|
||||
lines.push('Description: Analisis traffic per agent berdasarkan MAC address dari flows table');
|
||||
lines.push(separator('─'));
|
||||
|
||||
const AGENT_MAC_MAP = {
|
||||
'2F-TF-1D-GK': { label: 'JRP Cibubur', macs: ['60:be:b4:1f:05:96'] },
|
||||
'8A-V3-PB-85': { label: 'IFG LT.18', macs: ['04:f4:1c:ce:c2:e6'] },
|
||||
'F6-2V-DT-8A': { label: 'CPI Balaraja', macs: ['2c:7b:a0:d8:86:91', '16:11:ac:73:34:1d', 'bc:45:5b:ca:d5:be', 'de:ed:cc:57:58:34', 'f4:6d:3f:ef:01:a0', '60:be:b4:29:d3:36'] },
|
||||
};
|
||||
|
||||
for (const [uuid, agent] of Object.entries(AGENT_MAC_MAP)) {
|
||||
lines.push('');
|
||||
lines.push(`Agent: ${agent.label} (${uuid})`);
|
||||
lines.push(`MACs: ${agent.macs.join(', ')}`);
|
||||
lines.push(separator('─', 40));
|
||||
|
||||
const ph = agent.macs.map(() => '?').join(',');
|
||||
|
||||
// Summary
|
||||
const sumRow = d.prepare(`
|
||||
SELECT COUNT(DISTINCT src_ip) AS device_count, COUNT(*) AS flow_count,
|
||||
SUM(bytes_download) AS total_dl, SUM(bytes_upload) AS total_ul
|
||||
FROM flows WHERE src_mac IN (${ph})
|
||||
`).get(...agent.macs);
|
||||
|
||||
lines.push(` Devices: ${fmtNum(sumRow.device_count)}`);
|
||||
lines.push(` Total Flows: ${fmtNum(sumRow.flow_count)}`);
|
||||
lines.push(` Total Download: ${fmtBytes(sumRow.total_dl)}`);
|
||||
lines.push(` Total Upload: ${fmtBytes(sumRow.total_ul)}`);
|
||||
|
||||
// Top apps
|
||||
const apps = d.prepare(`
|
||||
SELECT app_label, SUM(bytes_download) AS dl, SUM(bytes_upload) AS ul, COUNT(*) AS cnt
|
||||
FROM flows WHERE src_mac IN (${ph}) AND app_label IS NOT NULL
|
||||
GROUP BY app_label ORDER BY dl DESC LIMIT 10
|
||||
`).all(...agent.macs);
|
||||
|
||||
lines.push(` Top Applications:`);
|
||||
apps.forEach((a, i) => {
|
||||
lines.push(` ${String(i+1).padStart(2)}. ${(a.app_label||'?').padEnd(30)} DL:${fmtBytes(a.dl).padStart(12)} UL:${fmtBytes(a.ul).padStart(12)} Flows:${a.cnt}`);
|
||||
});
|
||||
|
||||
// Top devices
|
||||
const devs = d.prepare(`
|
||||
SELECT src_ip, SUM(bytes_download) AS dl, MAX(last_seen) AS last
|
||||
FROM flows WHERE src_mac IN (${ph})
|
||||
GROUP BY src_ip ORDER BY dl DESC LIMIT 10
|
||||
`).all(...agent.macs);
|
||||
|
||||
lines.push(` Top Devices:`);
|
||||
devs.forEach((d2, i) => {
|
||||
lines.push(` ${String(i+1).padStart(2)}. ${(d2.src_ip||'?').padEnd(20)} DL:${fmtBytes(d2.dl).padStart(12)} Last:${d2.last||'-'}`);
|
||||
});
|
||||
}
|
||||
|
||||
// ── Bandwidth Apps Summary ─────────────────────────────────────────────────
|
||||
lines.push('');
|
||||
lines.push(separator('═'));
|
||||
lines.push('[DERIVED: BANDWIDTH APPS LATEST SNAPSHOT]');
|
||||
lines.push('Description: Snapshot terakhir bandwidth per aplikasi (nilai aktual, bukan akumulasi)');
|
||||
lines.push(separator('─'));
|
||||
|
||||
const latestBwSnap = d.prepare('SELECT MAX(fetched_at) as t FROM bandwidth_apps').get()?.t;
|
||||
if (latestBwSnap) {
|
||||
lines.push(`Latest Snapshot: ${latestBwSnap}`);
|
||||
const bwApps = d.prepare('SELECT app_label, category, download, upload, total, flow_count FROM bandwidth_apps WHERE fetched_at = ? ORDER BY download DESC').all(latestBwSnap);
|
||||
lines.push(`Total Apps: ${bwApps.length}`);
|
||||
lines.push('');
|
||||
bwApps.forEach((a, i) => {
|
||||
lines.push(` ${String(i+1).padStart(3)}. ${(a.app_label||'?').padEnd(30)} [${(a.category||'?').padEnd(20)}] DL:${fmtBytes(a.download).padStart(12)} UL:${fmtBytes(a.upload).padStart(12)} Flows:${fmtNum(a.flow_count)}`);
|
||||
});
|
||||
}
|
||||
|
||||
// ── Encryption Audit Summary ───────────────────────────────────────────────
|
||||
lines.push('');
|
||||
lines.push(separator('═'));
|
||||
lines.push('[DERIVED: ENCRYPTION RISK SUMMARY]');
|
||||
lines.push('Description: Distribusi risk level enkripsi per perangkat (snapshot terbaru)');
|
||||
lines.push(separator('─'));
|
||||
|
||||
const latestEncSnap = d.prepare('SELECT MAX(fetched_at) as t FROM intel_encryption_audit').get()?.t;
|
||||
if (latestEncSnap) {
|
||||
const riskDist = d.prepare(`
|
||||
SELECT risk_level, COUNT(*) as cnt, AVG(encrypted_pct) as avg_enc
|
||||
FROM intel_encryption_audit WHERE fetched_at = ?
|
||||
GROUP BY risk_level ORDER BY cnt DESC
|
||||
`).all(latestEncSnap);
|
||||
|
||||
lines.push(`Latest Snapshot: ${latestEncSnap}`);
|
||||
lines.push('Risk Distribution:');
|
||||
riskDist.forEach(r => {
|
||||
lines.push(` ${(r.risk_level||'Unknown').padEnd(15)} ${String(r.cnt).padStart(5)} devices avg encrypted: ${Number(r.avg_enc||0).toFixed(1)}%`);
|
||||
});
|
||||
|
||||
// Highest risk devices
|
||||
lines.push('');
|
||||
lines.push('Critical Risk Devices (0% encrypted):');
|
||||
const critDevs = d.prepare(`
|
||||
SELECT ip_address, mac_address, device_label, encrypted_pct, total
|
||||
FROM intel_encryption_audit WHERE fetched_at = ? AND risk_level = 'Critical'
|
||||
ORDER BY total DESC LIMIT 20
|
||||
`).all(latestEncSnap);
|
||||
critDevs.forEach(r => {
|
||||
lines.push(JSON.stringify(r));
|
||||
});
|
||||
}
|
||||
|
||||
// ── DNS Top Domains ────────────────────────────────────────────────────────
|
||||
lines.push('');
|
||||
lines.push(separator('═'));
|
||||
lines.push('[DERIVED: TOP DNS DOMAINS]');
|
||||
lines.push('Description: Domain paling sering diquery dari DNS stats');
|
||||
lines.push(separator('─'));
|
||||
|
||||
const latestDnsSnap = d.prepare('SELECT MAX(fetched_at) as t FROM dns_queries').get()?.t;
|
||||
if (latestDnsSnap) {
|
||||
const dnsRows = d.prepare('SELECT * FROM dns_queries WHERE fetched_at = ? ORDER BY query_count DESC LIMIT 30').all(latestDnsSnap);
|
||||
|
||||
lines.push(`Latest Snapshot: ${latestDnsSnap}`);
|
||||
dnsRows.forEach(r => lines.push(JSON.stringify(r)));
|
||||
}
|
||||
|
||||
// ── IP Reputation Blacklisted ─────────────────────────────────────────────
|
||||
lines.push('');
|
||||
lines.push(separator('═'));
|
||||
lines.push('[DERIVED: BLACKLISTED IP ADDRESSES]');
|
||||
lines.push('Description: IP address yang terdeteksi blacklisted (dari intel_ip_reputation)');
|
||||
lines.push(separator('─'));
|
||||
|
||||
const latestRepSnap = d.prepare('SELECT MAX(fetched_at) as t FROM intel_ip_reputation').get()?.t;
|
||||
if (latestRepSnap) {
|
||||
const blacklisted = d.prepare('SELECT * FROM intel_ip_reputation WHERE fetched_at = ? AND blacklisted = 1').all(latestRepSnap);
|
||||
lines.push(`Latest Snapshot: ${latestRepSnap}`);
|
||||
lines.push(`Blacklisted count: ${blacklisted.length}`);
|
||||
blacklisted.forEach(r => lines.push(JSON.stringify(r)));
|
||||
}
|
||||
|
||||
// ── Flows: Active Sessions Summary ────────────────────────────────────────
|
||||
lines.push('');
|
||||
lines.push(separator('═'));
|
||||
lines.push('[DERIVED: ACTIVE FLOWS SUMMARY]');
|
||||
lines.push('Description: Ringkasan aliran jaringan aktif (50 terbaru per download)');
|
||||
lines.push(separator('─'));
|
||||
|
||||
const flowSummary = d.prepare(`
|
||||
SELECT COUNT(*) as total_flows,
|
||||
COUNT(DISTINCT src_ip) as unique_src_ips,
|
||||
COUNT(DISTINCT dst_ip) as unique_dst_ips,
|
||||
COUNT(DISTINCT src_mac) as unique_macs,
|
||||
SUM(bytes_download) as total_dl,
|
||||
SUM(bytes_upload) as total_ul,
|
||||
MIN(first_seen) as earliest,
|
||||
MAX(last_seen) as latest
|
||||
FROM flows
|
||||
`).get();
|
||||
|
||||
lines.push(`Total Flows in DB: ${fmtNum(flowSummary.total_flows)}`);
|
||||
lines.push(`Unique Source IPs: ${fmtNum(flowSummary.unique_src_ips)}`);
|
||||
lines.push(`Unique Dest IPs: ${fmtNum(flowSummary.unique_dst_ips)}`);
|
||||
lines.push(`Unique MAC Addresses: ${fmtNum(flowSummary.unique_macs)}`);
|
||||
lines.push(`Total Download: ${fmtBytes(flowSummary.total_dl)}`);
|
||||
lines.push(`Total Upload: ${fmtBytes(flowSummary.total_ul)}`);
|
||||
lines.push(`Data from: ${flowSummary.earliest}`);
|
||||
lines.push(`Data to: ${flowSummary.latest}`);
|
||||
lines.push('');
|
||||
|
||||
// Top 50 flows by download
|
||||
lines.push('Top 50 Flows by Download:');
|
||||
const topFlows = d.prepare('SELECT * FROM flows ORDER BY bytes_download DESC LIMIT 50').all();
|
||||
topFlows.forEach(r => lines.push(JSON.stringify(r)));
|
||||
|
||||
// ── Unencrypted Password Events ───────────────────────────────────────────
|
||||
lines.push('');
|
||||
lines.push(separator('═'));
|
||||
lines.push('[DERIVED: UNENCRYPTED PASSWORD DETECTIONS]');
|
||||
lines.push('Description: Kejadian pengiriman credential dalam plaintext (HIGH severity)');
|
||||
lines.push(separator('─'));
|
||||
|
||||
const unencPwdHigh = d.prepare(`
|
||||
SELECT ip_address, mac_address, dst_ip, dst_port, protocol, username, download, upload, severity, detected_at
|
||||
FROM intel_unencrypted_passwords ORDER BY detected_at DESC
|
||||
`).all();
|
||||
lines.push(`Total detections: ${unencPwdHigh.length}`);
|
||||
unencPwdHigh.forEach(r => lines.push(JSON.stringify(r)));
|
||||
|
||||
// ── Footer ────────────────────────────────────────────────────────────────
|
||||
lines.push('');
|
||||
lines.push(separator('═'));
|
||||
lines.push(' END OF EXPORT');
|
||||
lines.push(` Generated at: ${new Date().toISOString()}`);
|
||||
lines.push(` Total lines: ${lines.length + 3}`);
|
||||
lines.push(separator('═'));
|
||||
|
||||
// Write to file
|
||||
const output = lines.join('\n');
|
||||
fs.writeFileSync(OUTPUT_FILE, output, 'utf-8');
|
||||
|
||||
const stats = fs.statSync(OUTPUT_FILE);
|
||||
console.log(`\n✅ Export selesai!`);
|
||||
console.log(` File: ${OUTPUT_FILE}`);
|
||||
console.log(` Size: ${fmtBytes(stats.size)}`);
|
||||
console.log(` Lines: ${fmtNum(lines.length)}`);
|
||||
console.log(` Tables: ${tables.length}`);
|
||||
console.log(` Total Rows: ${fmtNum(totalRows)}`);
|
||||
}
|
||||
|
||||
main().catch(e => {
|
||||
console.error('❌ Export FAILED:', e);
|
||||
process.exit(1);
|
||||
});
|
||||
/**
|
||||
* generate_export.js
|
||||
*
|
||||
* Mengekspor SELURUH data dari semua tabel SQLite (database)
|
||||
* ke dalam file backone_data_export.txt
|
||||
*
|
||||
* Format output:
|
||||
* - Header metadata (tanggal, versi, jumlah tabel)
|
||||
* - Untuk setiap tabel: header section, row count, schema, dan semua data (JSON per baris)
|
||||
* - Footer summary
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const db = require('./database');
|
||||
|
||||
const OUTPUT_FILE = path.join(__dirname, '../backone_data_export.txt');
|
||||
const d = db.getDB();
|
||||
|
||||
// ─── Helpers ────────────────────────────────────────────────────────────────
|
||||
function fmtBytes(bytes) {
|
||||
if (!bytes || bytes === 0) return '0 B';
|
||||
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||
let b = Math.abs(bytes);
|
||||
let i = 0;
|
||||
while (b >= 1024 && i < units.length - 1) { b /= 1024; i++; }
|
||||
return b.toFixed(2) + ' ' + units[i];
|
||||
}
|
||||
|
||||
function fmtNum(n) {
|
||||
if (n == null) return 'N/A';
|
||||
return Number(n).toLocaleString('id-ID');
|
||||
}
|
||||
|
||||
function separator(char = '═', len = 80) {
|
||||
return char.repeat(len);
|
||||
}
|
||||
|
||||
function sectionHeader(tableName, rowCount, description) {
|
||||
return [
|
||||
'',
|
||||
separator('═'),
|
||||
`[TABLE: ${tableName}]`,
|
||||
`Row Count: ${fmtNum(rowCount)}`,
|
||||
description ? `Description: ${description}` : '',
|
||||
separator('─'),
|
||||
].filter(l => l !== '').join('\n');
|
||||
}
|
||||
|
||||
// ─── Table descriptions ──────────────────────────────────────────────────────
|
||||
const TABLE_DESCRIPTIONS = {
|
||||
bandwidth_apps : 'Bandwidth per aplikasi (YouTube, Facebook, dll) dari BackOne DPI',
|
||||
bandwidth_timeline : 'Timeline bandwidth per menit (download/upload historis)',
|
||||
bittorrent_info_hashes: 'Deteksi aktivitas BitTorrent berdasarkan info hash',
|
||||
countries : 'Distribusi traffic berdasarkan negara tujuan',
|
||||
devices : 'Daftar perangkat (IP/MAC) beserta bandwidth & OS',
|
||||
dhcp_fingerprints : 'Fingerprint DHCP untuk identifikasi tipe device',
|
||||
discovered_os : 'OS yang terdeteksi dari traffic scanning',
|
||||
dns_stats : 'Query DNS teratas dan statistik resolusi domain',
|
||||
events : 'Event log dari BackOne agent (koneksi, peringatan, dll)',
|
||||
flows : 'Data aliran jaringan per-sesi (src IP, dst IP, aplikasi, domain, bytes)',
|
||||
flow_origins : 'Asal flow: lokal (LAN) atau eksternal (WAN)',
|
||||
flow_types : 'Tipe flow: TCP, UDP, ICMP, dll',
|
||||
http_user_agents : 'HTTP User-Agent yang terdeteksi (browser, OS, framework)',
|
||||
intel_crypto_mining : 'Deteksi aktivitas crypto mining (pool host, protokol)',
|
||||
intel_device_discovery: 'Penemuan perangkat baru di jaringan (tipe, OS, manufaktur)',
|
||||
intel_encryption_audit: 'Audit enkripsi traffic per perangkat (encrypted%, risk level)',
|
||||
intel_insecure_protocols: 'Protokol tidak aman yang terdeteksi (HTTP, Telnet, FTP, dll)',
|
||||
intel_ip_reputation : 'Reputasi IP eksternal (blacklist, threat score)',
|
||||
intel_server_discovery: 'Server yang terdeteksi (HTTPS, SSH, HTTP, dll)',
|
||||
intel_tor_detection : 'Deteksi penggunaan jaringan Tor',
|
||||
intel_unencrypted_passwords: 'Deteksi pengiriman password dalam bentuk plaintext',
|
||||
intel_vpn_detection : 'Deteksi penggunaan VPN (OpenVPN, WireGuard, dll)',
|
||||
interfaces : 'Interface jaringan per agent (WAN/LAN, bandwidth)',
|
||||
ip_versions : 'Distribusi traffic IPv4 vs IPv6',
|
||||
mac_bandwidth : 'Bandwidth per MAC address perangkat',
|
||||
mdns_hostnames : 'mDNS hostname yang terdeteksi di jaringan lokal',
|
||||
netbios_hostnames : 'NetBIOS hostname (nama komputer Windows)',
|
||||
protocols : 'Distribusi protokol jaringan (port usage)',
|
||||
quic_hostnames : 'Hostname via QUIC/HTTP3 (Google, Cloudflare, dll)',
|
||||
regions : 'Distribusi traffic berdasarkan region/kota tujuan',
|
||||
remote_ips : 'IP remote teratas yang diakses perangkat',
|
||||
sni_hostnames : 'Server Name Indication dari koneksi TLS',
|
||||
ssh_versions : 'Versi SSH yang terdeteksi di jaringan',
|
||||
ssl_server_cn : 'Common Name sertifikat SSL server',
|
||||
threats : 'Ancaman keamanan terdeteksi (threat alerts)',
|
||||
tls_ciphers : 'Cipher suite TLS yang digunakan',
|
||||
tls_security : 'Tingkat keamanan TLS (Modern, Compatible, Old)',
|
||||
tls_versions : 'Versi TLS yang digunakan (1.0, 1.2, 1.3)',
|
||||
vlans : 'VLAN yang terdeteksi di jaringan',
|
||||
};
|
||||
|
||||
// ─── Main Export Logic ───────────────────────────────────────────────────────
|
||||
async function main() {
|
||||
console.log('🚀 Memulai export data...');
|
||||
|
||||
const exportDate = new Date().toISOString();
|
||||
const lines = [];
|
||||
|
||||
// ── File Header ──────────────────────────────────────────────────────────
|
||||
lines.push(separator('═'));
|
||||
lines.push(' BACKONE DATA EXPORT');
|
||||
lines.push(' Seluruh data hasil parsing dari BackOne API');
|
||||
lines.push(separator('─'));
|
||||
lines.push(` Export Date: ${exportDate}`);
|
||||
lines.push(` Generated by: generate_export.js`);
|
||||
lines.push(` Source: database (SQLite lokal)`);
|
||||
lines.push(` API Base: BackOne API Service`);
|
||||
lines.push(` Format: Per-tabel, data JSON satu record per baris (JSONL)`);
|
||||
lines.push(separator('─'));
|
||||
|
||||
// ── Get all tables ────────────────────────────────────────────────────────
|
||||
const tables = d.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
|
||||
).all().map(r => r.name);
|
||||
|
||||
lines.push(` Total Tables: ${tables.length}`);
|
||||
lines.push(separator('═'));
|
||||
lines.push('');
|
||||
|
||||
// ── Table of Contents ─────────────────────────────────────────────────────
|
||||
lines.push('TABLE OF CONTENTS');
|
||||
lines.push(separator('─', 40));
|
||||
let totalRows = 0;
|
||||
const tableSummaries = [];
|
||||
for (const tableName of tables) {
|
||||
const cnt = d.prepare(`SELECT COUNT(*) as c FROM ${tableName}`).get().c;
|
||||
totalRows += cnt;
|
||||
const desc = TABLE_DESCRIPTIONS[tableName] || '-';
|
||||
lines.push(` ${tableName.padEnd(35)} ${String(cnt).padStart(8)} rows`);
|
||||
tableSummaries.push({ name: tableName, count: cnt, description: desc });
|
||||
}
|
||||
lines.push(separator('─', 40));
|
||||
lines.push(` ${'TOTAL'.padEnd(35)} ${String(totalRows).padStart(8)} rows`);
|
||||
lines.push('');
|
||||
|
||||
// ── Per-Table Export ──────────────────────────────────────────────────────
|
||||
for (const { name: tableName, count, description } of tableSummaries) {
|
||||
console.log(` 📋 Exporting: ${tableName} (${fmtNum(count)} rows)...`);
|
||||
|
||||
// Section header
|
||||
lines.push(sectionHeader(tableName, count, description));
|
||||
|
||||
// Schema
|
||||
const cols = d.prepare(`PRAGMA table_info(${tableName})`).all();
|
||||
lines.push('Schema:');
|
||||
cols.forEach(c => {
|
||||
lines.push(` - ${c.name} [${c.type || 'TEXT'}]${c.notnull ? ' NOT NULL' : ''}${c.pk ? ' PRIMARY KEY' : ''}`);
|
||||
});
|
||||
lines.push('');
|
||||
|
||||
// Statistics for numeric columns
|
||||
const numericCols = cols.filter(c =>
|
||||
['INTEGER', 'REAL', 'NUMERIC'].includes((c.type || '').toUpperCase()) &&
|
||||
!['id'].includes(c.name.toLowerCase())
|
||||
);
|
||||
|
||||
if (count > 0 && numericCols.length > 0) {
|
||||
lines.push('Statistics:');
|
||||
for (const col of numericCols.slice(0, 5)) { // max 5 numeric cols
|
||||
try {
|
||||
const stat = d.prepare(`
|
||||
SELECT MIN(${col.name}) as min, MAX(${col.name}) as max,
|
||||
AVG(${col.name}) as avg, SUM(${col.name}) as total
|
||||
FROM ${tableName}
|
||||
`).get();
|
||||
if (stat && stat.max !== null) {
|
||||
lines.push(` ${col.name}: min=${fmtNum(stat.min)} max=${fmtNum(stat.max)} avg=${Number(stat.avg || 0).toFixed(2)} total=${fmtNum(stat.total)}`);
|
||||
}
|
||||
} catch(e) { /* skip */ }
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Data rows (ALL rows)
|
||||
if (count === 0) {
|
||||
lines.push('(No data)');
|
||||
} else {
|
||||
lines.push(`Data (${fmtNum(count)} records):`);
|
||||
const rows = d.prepare(`SELECT * FROM ${tableName}`).all();
|
||||
for (const row of rows) {
|
||||
lines.push(JSON.stringify(row));
|
||||
}
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// ── Agent-specific sections (derived from flows) ───────────────────────────
|
||||
lines.push('');
|
||||
lines.push(separator('═'));
|
||||
lines.push('[DERIVED: AGENT ANALYSIS]');
|
||||
lines.push('Description: Analisis traffic per agent berdasarkan MAC address dari flows table');
|
||||
lines.push(separator('─'));
|
||||
|
||||
const AGENT_MAC_MAP = {
|
||||
'2F-TF-1D-GK': { label: 'JRP Cibubur', macs: ['60:be:b4:1f:05:96'] },
|
||||
'8A-V3-PB-85': { label: 'IFG LT.18', macs: ['04:f4:1c:ce:c2:e6'] },
|
||||
'F6-2V-DT-8A': { label: 'CPI Balaraja', macs: ['2c:7b:a0:d8:86:91', '16:11:ac:73:34:1d', 'bc:45:5b:ca:d5:be', 'de:ed:cc:57:58:34', 'f4:6d:3f:ef:01:a0', '60:be:b4:29:d3:36'] },
|
||||
};
|
||||
|
||||
for (const [uuid, agent] of Object.entries(AGENT_MAC_MAP)) {
|
||||
lines.push('');
|
||||
lines.push(`Agent: ${agent.label} (${uuid})`);
|
||||
lines.push(`MACs: ${agent.macs.join(', ')}`);
|
||||
lines.push(separator('─', 40));
|
||||
|
||||
const ph = agent.macs.map(() => '?').join(',');
|
||||
|
||||
// Summary
|
||||
const sumRow = d.prepare(`
|
||||
SELECT COUNT(DISTINCT src_ip) AS device_count, COUNT(*) AS flow_count,
|
||||
SUM(bytes_download) AS total_dl, SUM(bytes_upload) AS total_ul
|
||||
FROM flows WHERE src_mac IN (${ph})
|
||||
`).get(...agent.macs);
|
||||
|
||||
lines.push(` Devices: ${fmtNum(sumRow.device_count)}`);
|
||||
lines.push(` Total Flows: ${fmtNum(sumRow.flow_count)}`);
|
||||
lines.push(` Total Download: ${fmtBytes(sumRow.total_dl)}`);
|
||||
lines.push(` Total Upload: ${fmtBytes(sumRow.total_ul)}`);
|
||||
|
||||
// Top apps
|
||||
const apps = d.prepare(`
|
||||
SELECT app_label, SUM(bytes_download) AS dl, SUM(bytes_upload) AS ul, COUNT(*) AS cnt
|
||||
FROM flows WHERE src_mac IN (${ph}) AND app_label IS NOT NULL
|
||||
GROUP BY app_label ORDER BY dl DESC LIMIT 10
|
||||
`).all(...agent.macs);
|
||||
|
||||
lines.push(` Top Applications:`);
|
||||
apps.forEach((a, i) => {
|
||||
lines.push(` ${String(i+1).padStart(2)}. ${(a.app_label||'?').padEnd(30)} DL:${fmtBytes(a.dl).padStart(12)} UL:${fmtBytes(a.ul).padStart(12)} Flows:${a.cnt}`);
|
||||
});
|
||||
|
||||
// Top devices
|
||||
const devs = d.prepare(`
|
||||
SELECT src_ip, SUM(bytes_download) AS dl, MAX(last_seen) AS last
|
||||
FROM flows WHERE src_mac IN (${ph})
|
||||
GROUP BY src_ip ORDER BY dl DESC LIMIT 10
|
||||
`).all(...agent.macs);
|
||||
|
||||
lines.push(` Top Devices:`);
|
||||
devs.forEach((d2, i) => {
|
||||
lines.push(` ${String(i+1).padStart(2)}. ${(d2.src_ip||'?').padEnd(20)} DL:${fmtBytes(d2.dl).padStart(12)} Last:${d2.last||'-'}`);
|
||||
});
|
||||
}
|
||||
|
||||
// ── Bandwidth Apps Summary ─────────────────────────────────────────────────
|
||||
lines.push('');
|
||||
lines.push(separator('═'));
|
||||
lines.push('[DERIVED: BANDWIDTH APPS LATEST SNAPSHOT]');
|
||||
lines.push('Description: Snapshot terakhir bandwidth per aplikasi (nilai aktual, bukan akumulasi)');
|
||||
lines.push(separator('─'));
|
||||
|
||||
const latestBwSnap = d.prepare('SELECT MAX(fetched_at) as t FROM bandwidth_apps').get()?.t;
|
||||
if (latestBwSnap) {
|
||||
lines.push(`Latest Snapshot: ${latestBwSnap}`);
|
||||
const bwApps = d.prepare('SELECT app_label, category, download, upload, total, flow_count FROM bandwidth_apps WHERE fetched_at = ? ORDER BY download DESC').all(latestBwSnap);
|
||||
lines.push(`Total Apps: ${bwApps.length}`);
|
||||
lines.push('');
|
||||
bwApps.forEach((a, i) => {
|
||||
lines.push(` ${String(i+1).padStart(3)}. ${(a.app_label||'?').padEnd(30)} [${(a.category||'?').padEnd(20)}] DL:${fmtBytes(a.download).padStart(12)} UL:${fmtBytes(a.upload).padStart(12)} Flows:${fmtNum(a.flow_count)}`);
|
||||
});
|
||||
}
|
||||
|
||||
// ── Encryption Audit Summary ───────────────────────────────────────────────
|
||||
lines.push('');
|
||||
lines.push(separator('═'));
|
||||
lines.push('[DERIVED: ENCRYPTION RISK SUMMARY]');
|
||||
lines.push('Description: Distribusi risk level enkripsi per perangkat (snapshot terbaru)');
|
||||
lines.push(separator('─'));
|
||||
|
||||
const latestEncSnap = d.prepare('SELECT MAX(fetched_at) as t FROM intel_encryption_audit').get()?.t;
|
||||
if (latestEncSnap) {
|
||||
const riskDist = d.prepare(`
|
||||
SELECT risk_level, COUNT(*) as cnt, AVG(encrypted_pct) as avg_enc
|
||||
FROM intel_encryption_audit WHERE fetched_at = ?
|
||||
GROUP BY risk_level ORDER BY cnt DESC
|
||||
`).all(latestEncSnap);
|
||||
|
||||
lines.push(`Latest Snapshot: ${latestEncSnap}`);
|
||||
lines.push('Risk Distribution:');
|
||||
riskDist.forEach(r => {
|
||||
lines.push(` ${(r.risk_level||'Unknown').padEnd(15)} ${String(r.cnt).padStart(5)} devices avg encrypted: ${Number(r.avg_enc||0).toFixed(1)}%`);
|
||||
});
|
||||
|
||||
// Highest risk devices
|
||||
lines.push('');
|
||||
lines.push('Critical Risk Devices (0% encrypted):');
|
||||
const critDevs = d.prepare(`
|
||||
SELECT ip_address, mac_address, device_label, encrypted_pct, total
|
||||
FROM intel_encryption_audit WHERE fetched_at = ? AND risk_level = 'Critical'
|
||||
ORDER BY total DESC LIMIT 20
|
||||
`).all(latestEncSnap);
|
||||
critDevs.forEach(r => {
|
||||
lines.push(JSON.stringify(r));
|
||||
});
|
||||
}
|
||||
|
||||
// ── DNS Top Domains ────────────────────────────────────────────────────────
|
||||
lines.push('');
|
||||
lines.push(separator('═'));
|
||||
lines.push('[DERIVED: TOP DNS DOMAINS]');
|
||||
lines.push('Description: Domain paling sering diquery dari DNS stats');
|
||||
lines.push(separator('─'));
|
||||
|
||||
const latestDnsSnap = d.prepare('SELECT MAX(fetched_at) as t FROM dns_queries').get()?.t;
|
||||
if (latestDnsSnap) {
|
||||
const dnsRows = d.prepare('SELECT * FROM dns_queries WHERE fetched_at = ? ORDER BY query_count DESC LIMIT 30').all(latestDnsSnap);
|
||||
|
||||
lines.push(`Latest Snapshot: ${latestDnsSnap}`);
|
||||
dnsRows.forEach(r => lines.push(JSON.stringify(r)));
|
||||
}
|
||||
|
||||
// ── IP Reputation Blacklisted ─────────────────────────────────────────────
|
||||
lines.push('');
|
||||
lines.push(separator('═'));
|
||||
lines.push('[DERIVED: BLACKLISTED IP ADDRESSES]');
|
||||
lines.push('Description: IP address yang terdeteksi blacklisted (dari intel_ip_reputation)');
|
||||
lines.push(separator('─'));
|
||||
|
||||
const latestRepSnap = d.prepare('SELECT MAX(fetched_at) as t FROM intel_ip_reputation').get()?.t;
|
||||
if (latestRepSnap) {
|
||||
const blacklisted = d.prepare('SELECT * FROM intel_ip_reputation WHERE fetched_at = ? AND blacklisted = 1').all(latestRepSnap);
|
||||
lines.push(`Latest Snapshot: ${latestRepSnap}`);
|
||||
lines.push(`Blacklisted count: ${blacklisted.length}`);
|
||||
blacklisted.forEach(r => lines.push(JSON.stringify(r)));
|
||||
}
|
||||
|
||||
// ── Flows: Active Sessions Summary ────────────────────────────────────────
|
||||
lines.push('');
|
||||
lines.push(separator('═'));
|
||||
lines.push('[DERIVED: ACTIVE FLOWS SUMMARY]');
|
||||
lines.push('Description: Ringkasan aliran jaringan aktif (50 terbaru per download)');
|
||||
lines.push(separator('─'));
|
||||
|
||||
const flowSummary = d.prepare(`
|
||||
SELECT COUNT(*) as total_flows,
|
||||
COUNT(DISTINCT src_ip) as unique_src_ips,
|
||||
COUNT(DISTINCT dst_ip) as unique_dst_ips,
|
||||
COUNT(DISTINCT src_mac) as unique_macs,
|
||||
SUM(bytes_download) as total_dl,
|
||||
SUM(bytes_upload) as total_ul,
|
||||
MIN(first_seen) as earliest,
|
||||
MAX(last_seen) as latest
|
||||
FROM flows
|
||||
`).get();
|
||||
|
||||
lines.push(`Total Flows in DB: ${fmtNum(flowSummary.total_flows)}`);
|
||||
lines.push(`Unique Source IPs: ${fmtNum(flowSummary.unique_src_ips)}`);
|
||||
lines.push(`Unique Dest IPs: ${fmtNum(flowSummary.unique_dst_ips)}`);
|
||||
lines.push(`Unique MAC Addresses: ${fmtNum(flowSummary.unique_macs)}`);
|
||||
lines.push(`Total Download: ${fmtBytes(flowSummary.total_dl)}`);
|
||||
lines.push(`Total Upload: ${fmtBytes(flowSummary.total_ul)}`);
|
||||
lines.push(`Data from: ${flowSummary.earliest}`);
|
||||
lines.push(`Data to: ${flowSummary.latest}`);
|
||||
lines.push('');
|
||||
|
||||
// Top 50 flows by download
|
||||
lines.push('Top 50 Flows by Download:');
|
||||
const topFlows = d.prepare('SELECT * FROM flows ORDER BY bytes_download DESC LIMIT 50').all();
|
||||
topFlows.forEach(r => lines.push(JSON.stringify(r)));
|
||||
|
||||
// ── Unencrypted Password Events ───────────────────────────────────────────
|
||||
lines.push('');
|
||||
lines.push(separator('═'));
|
||||
lines.push('[DERIVED: UNENCRYPTED PASSWORD DETECTIONS]');
|
||||
lines.push('Description: Kejadian pengiriman credential dalam plaintext (HIGH severity)');
|
||||
lines.push(separator('─'));
|
||||
|
||||
const unencPwdHigh = d.prepare(`
|
||||
SELECT ip_address, mac_address, dst_ip, dst_port, protocol, username, download, upload, severity, detected_at
|
||||
FROM intel_unencrypted_passwords ORDER BY detected_at DESC
|
||||
`).all();
|
||||
lines.push(`Total detections: ${unencPwdHigh.length}`);
|
||||
unencPwdHigh.forEach(r => lines.push(JSON.stringify(r)));
|
||||
|
||||
// ── Footer ────────────────────────────────────────────────────────────────
|
||||
lines.push('');
|
||||
lines.push(separator('═'));
|
||||
lines.push(' END OF EXPORT');
|
||||
lines.push(` Generated at: ${new Date().toISOString()}`);
|
||||
lines.push(` Total lines: ${lines.length + 3}`);
|
||||
lines.push(separator('═'));
|
||||
|
||||
// Write to file
|
||||
const output = lines.join('\n');
|
||||
fs.writeFileSync(OUTPUT_FILE, output, 'utf-8');
|
||||
|
||||
const stats = fs.statSync(OUTPUT_FILE);
|
||||
console.log(`\n✅ Export selesai!`);
|
||||
console.log(` File: ${OUTPUT_FILE}`);
|
||||
console.log(` Size: ${fmtBytes(stats.size)}`);
|
||||
console.log(` Lines: ${fmtNum(lines.length)}`);
|
||||
console.log(` Tables: ${tables.length}`);
|
||||
console.log(` Total Rows: ${fmtNum(totalRows)}`);
|
||||
}
|
||||
|
||||
main().catch(e => {
|
||||
console.error('❌ Export FAILED:', e);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
const jwt = require('jsonwebtoken');
|
||||
const User = require('../models/User');
|
||||
const Session = require('../models/Session');
|
||||
const { Summary } = require('../models/Schemas');
|
||||
|
||||
const JWT_SECRET = process.env.JWT_SECRET || 'super-secret-backone-key';
|
||||
@@ -11,19 +12,46 @@ async function requireAuth(req, res, next) {
|
||||
try {
|
||||
req.user = jwt.verify(token, JWT_SECRET);
|
||||
|
||||
// Verify session status in MongoDB
|
||||
if (req.user.session_id) {
|
||||
const activeSession = await Session.findById(req.user.session_id);
|
||||
if (!activeSession) {
|
||||
res.clearCookie('token');
|
||||
return res.status(401).json({ error: 'Sesi login telah dinonaktifkan atau kedaluwarsa.' });
|
||||
}
|
||||
// Update last active
|
||||
activeSession.last_active = new Date();
|
||||
await activeSession.save();
|
||||
}
|
||||
|
||||
// ── VIEW-AS MODE ──────────────────────────────────────────────────────────
|
||||
const viewAsHeader = req.headers['x-view-as-agent'];
|
||||
if (viewAsHeader && (req.user.role === 'SUPER_ADMIN' || req.user.role === 'TENANT_ADMIN')) {
|
||||
const isAllowedViewAs = req.user.role === 'SUPER_ADMIN' ||
|
||||
req.user.role === 'TENANT_ADMIN' ||
|
||||
req.user.role === 'COMPANY_ADMIN' ||
|
||||
req.user.role === 'COMPANY_OPERATOR';
|
||||
|
||||
if (viewAsHeader && isAllowedViewAs) {
|
||||
try {
|
||||
const viewDecoded = jwt.verify(viewAsHeader, JWT_SECRET);
|
||||
if (viewDecoded.type === 'view-as' && viewDecoded.adminId === req.user.id && viewDecoded.viewAs) {
|
||||
const targetAgentUser = await User.findOne({ agent_uuid: viewDecoded.viewAs, role: 'AGENT_VIEWER' }).lean();
|
||||
const targetAgent = viewDecoded.viewAs;
|
||||
|
||||
// Validation: COMPANY_ADMIN and COMPANY_OPERATOR can only view-as their assigned agents
|
||||
if (['COMPANY_ADMIN', 'COMPANY_OPERATOR'].includes(req.user.role)) {
|
||||
const hasAccess = req.user.agent_uuids && req.user.agent_uuids.includes(targetAgent);
|
||||
if (!hasAccess) {
|
||||
throw new Error('Unauthorized view-as agent access');
|
||||
}
|
||||
}
|
||||
|
||||
const targetAgentUser = await User.findOne({ agent_uuid: targetAgent, role: 'AGENT_VIEWER' }).lean();
|
||||
|
||||
let targetSiteUuid = req.user.site_uuid;
|
||||
if (targetAgentUser && targetAgentUser.site_uuid) {
|
||||
targetSiteUuid = targetAgentUser.site_uuid;
|
||||
} else {
|
||||
const summaryDoc = await Summary.findOne({ agent_uuid: viewDecoded.viewAs }).lean();
|
||||
const summaryDoc = await Summary.findOne({ agent_uuid: targetAgent }).lean();
|
||||
if (summaryDoc && summaryDoc.site_uuid) {
|
||||
targetSiteUuid = summaryDoc.site_uuid;
|
||||
}
|
||||
@@ -32,7 +60,7 @@ async function requireAuth(req, res, next) {
|
||||
req.user = {
|
||||
...req.user,
|
||||
role: 'AGENT_VIEWER',
|
||||
agent_uuid: viewDecoded.viewAs,
|
||||
agent_uuid: targetAgent,
|
||||
agent_label: viewDecoded.viewAsLabel,
|
||||
site_uuid: targetSiteUuid,
|
||||
_viewAsMode: true,
|
||||
@@ -50,12 +78,25 @@ async function requireAuth(req, res, next) {
|
||||
}
|
||||
}
|
||||
|
||||
function requireAdmin(req, res, next) {
|
||||
async function requireAdmin(req, res, next) {
|
||||
const token = req.cookies?.token;
|
||||
if (!token) return res.status(401).json({ error: 'Not authenticated' });
|
||||
try {
|
||||
const decoded = jwt.verify(token, JWT_SECRET);
|
||||
if (decoded.role !== 'SUPER_ADMIN' && decoded.role !== 'TENANT_ADMIN' && decoded.role !== 'SOC_ANALYST') {
|
||||
|
||||
// Verify session status in MongoDB
|
||||
if (decoded.session_id) {
|
||||
const activeSession = await Session.findById(decoded.session_id);
|
||||
if (!activeSession) {
|
||||
res.clearCookie('token');
|
||||
return res.status(401).json({ error: 'Sesi login telah dinonaktifkan atau kedaluwarsa.' });
|
||||
}
|
||||
activeSession.last_active = new Date();
|
||||
await activeSession.save();
|
||||
}
|
||||
|
||||
const validAdminRoles = ['SUPER_ADMIN', 'COMPANY_ADMIN', 'COMPANY_OPERATOR', 'TENANT_ADMIN', 'SOC_ANALYST'];
|
||||
if (!validAdminRoles.includes(decoded.role)) {
|
||||
return res.status(403).json({ error: 'Forbidden' });
|
||||
}
|
||||
req.adminUser = decoded;
|
||||
|
||||
+187
-184
@@ -1,184 +1,187 @@
|
||||
// backend/models/Schemas.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// MongoDB Schemas untuk BackOne Backend (READ-ONLY)
|
||||
//
|
||||
// PENTING: Schema ini harus sinkron dengan proxy/models/Schemas.js
|
||||
// Proxy yang MENULIS data, backend yang MEMBACA data.
|
||||
//
|
||||
// Setiap dokumen di-tag dengan:
|
||||
// agent_uuid → identifikasi Network Agent spesifik (isolasi per tenant)
|
||||
// site_uuid → identifikasi site DPI (BackOne)
|
||||
// timestamp → waktu data dikumpulkan
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const mongoose = require('mongoose');
|
||||
|
||||
const baseOptions = {
|
||||
timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
|
||||
};
|
||||
|
||||
// ─── Bandwidth Summary (per agent, per collection cycle) ───────────────────────
|
||||
const SummarySchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true }, // null = global/all agents
|
||||
site_uuid: { type: String, index: true },
|
||||
bandwidth_down: Number,
|
||||
bandwidth_up: Number,
|
||||
active_flows: Number,
|
||||
download_speed: Number,
|
||||
upload_speed: Number,
|
||||
total_devices: Number,
|
||||
total_threats: Number,
|
||||
packet_drops: Number,
|
||||
peak_flow_rate: Number,
|
||||
cpu_usage: Number,
|
||||
memory_usage: Number,
|
||||
queue_depth: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Top Applications (per agent) ─────────────────────────────────────────────
|
||||
const AppStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
app_label: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Protocol Statistics (per agent) ──────────────────────────────────────────
|
||||
const ProtocolStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
protocol_label: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Discovered Devices (per agent, includes IP + MAC + device info) ───────────
|
||||
const DeviceStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
ip_address: { type: String, required: true, index: true },
|
||||
mac_address: { type: String, index: true },
|
||||
device_label: String,
|
||||
device_type: String,
|
||||
os_label: String,
|
||||
manufacturer: String,
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
last_seen: String,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Network Flows (per agent) ─────────────────────────────────────────────────
|
||||
const FlowSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
flow_id: String,
|
||||
src_ip: { type: String, index: true },
|
||||
src_mac: String,
|
||||
dst_ip: { type: String, index: true },
|
||||
dst_port: Number,
|
||||
protocol: String,
|
||||
app_label: String,
|
||||
domain: { type: String, index: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
first_seen: String,
|
||||
last_seen: String,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Cyber Threats (per agent) ─────────────────────────────────────────────────
|
||||
const ThreatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
threat_type: String,
|
||||
severity: String,
|
||||
src_ip: String,
|
||||
dst_ip: String,
|
||||
dst_port: Number,
|
||||
protocol: String,
|
||||
description: String,
|
||||
event_at: String,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── App Categories (per agent) ───────────────────────────────────────────────
|
||||
const AppCategoryStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
category_label: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── System Events (per agent) ─────────────────────────────────────────────────
|
||||
const EventSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
event_id: Number,
|
||||
event_type: String,
|
||||
severity: String,
|
||||
description: String,
|
||||
category_label: String,
|
||||
ip_address: String,
|
||||
mac_address: String,
|
||||
event_at: Date,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Compound Indexes for common dashboard queries ─────────────────────────────
|
||||
SummarySchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
AppStatSchema.index({ agent_uuid: 1, timestamp: -1, download: -1 });
|
||||
DeviceStatSchema.index({ agent_uuid: 1, ip_address: 1 }, { unique: true });
|
||||
FlowSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
FlowSchema.index({ agent_uuid: 1, flow_id: 1 });
|
||||
FlowSchema.index({ agent_uuid: 1, protocol: 1, timestamp: -1 });
|
||||
FlowSchema.index({ agent_uuid: 1, domain: 1, timestamp: -1 });
|
||||
FlowSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
|
||||
ThreatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
AppCategoryStatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
EventSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
|
||||
// ── Per-Device Per-Application Stats (synced from proxy) ─────────────────
|
||||
const DeviceAppStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
ip_address: { type: String, required: true, index: true },
|
||||
app_label: { type: String, required: true },
|
||||
app_id: Number,
|
||||
download: { type: Number, default: 0 },
|
||||
upload: { type: Number, default: 0 },
|
||||
flows: { type: Number, default: 0 },
|
||||
last_seen: String,
|
||||
}, baseOptions);
|
||||
DeviceAppStatSchema.index({ agent_uuid: 1, ip_address: 1, timestamp: -1 });
|
||||
DeviceAppStatSchema.index({ ip_address: 1, app_label: 1, timestamp: -1 });
|
||||
DeviceAppStatSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
|
||||
|
||||
const telemetrySchemas = require('./SchemasTelemetry');
|
||||
const auxSchemas = require('./SchemasAux');
|
||||
|
||||
module.exports = {
|
||||
Summary: mongoose.model('Summary', SummarySchema),
|
||||
AppStat: mongoose.model('AppStat', AppStatSchema),
|
||||
ProtocolStat: mongoose.model('ProtocolStat', ProtocolStatSchema),
|
||||
DeviceStat: mongoose.model('DeviceStat', DeviceStatSchema),
|
||||
DeviceAppStat: mongoose.model('DeviceAppStat', DeviceAppStatSchema),
|
||||
Flow: mongoose.model('Flow', FlowSchema),
|
||||
Threat: mongoose.model('Threat', ThreatSchema),
|
||||
AppCategoryStat: mongoose.model('AppCategoryStat', AppCategoryStatSchema),
|
||||
Event: mongoose.model('Event', EventSchema),
|
||||
...auxSchemas,
|
||||
...telemetrySchemas
|
||||
};
|
||||
|
||||
// backend/models/Schemas.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// MongoDB Schemas untuk BackOne Backend (READ-ONLY)
|
||||
//
|
||||
// PENTING: Schema ini harus sinkron dengan proxy/models/Schemas.js
|
||||
// Proxy yang MENULIS data, backend yang MEMBACA data.
|
||||
//
|
||||
// Setiap dokumen di-tag dengan:
|
||||
// agent_uuid → identifikasi Network Agent spesifik (isolasi per tenant)
|
||||
// site_uuid → identifikasi site DPI (BackOne)
|
||||
// timestamp → waktu data dikumpulkan
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const mongoose = require('mongoose');
|
||||
|
||||
const baseOptions = {
|
||||
timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
|
||||
};
|
||||
|
||||
// ─── Bandwidth Summary (per agent, per collection cycle) ───────────────────────
|
||||
const SummarySchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true }, // null = global/all agents
|
||||
site_uuid: { type: String, index: true },
|
||||
bandwidth_down: Number,
|
||||
bandwidth_up: Number,
|
||||
active_flows: Number,
|
||||
download_speed: Number,
|
||||
upload_speed: Number,
|
||||
total_devices: Number,
|
||||
total_threats: Number,
|
||||
packet_drops: Number,
|
||||
peak_flow_rate: Number,
|
||||
cpu_usage: Number,
|
||||
memory_usage: Number,
|
||||
queue_depth: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Top Applications (per agent) ─────────────────────────────────────────────
|
||||
const AppStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
app_label: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Protocol Statistics (per agent) ──────────────────────────────────────────
|
||||
const ProtocolStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
protocol_label: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Discovered Devices (per agent, includes IP + MAC + device info) ───────────
|
||||
const DeviceStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
ip_address: { type: String, required: true, index: true },
|
||||
mac_address: { type: String, index: true },
|
||||
device_label: String,
|
||||
device_type: String,
|
||||
os_label: String,
|
||||
manufacturer: String,
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
last_seen: String,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Network Flows (per agent) ─────────────────────────────────────────────────
|
||||
const FlowSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
flow_id: String,
|
||||
src_ip: { type: String, index: true },
|
||||
src_mac: { type: String, index: true },
|
||||
dst_ip: { type: String, index: true },
|
||||
dst_port: Number,
|
||||
protocol: String,
|
||||
app_label: String,
|
||||
domain: { type: String, index: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
first_seen: String,
|
||||
last_seen: String,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Cyber Threats (per agent) ─────────────────────────────────────────────────
|
||||
const ThreatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
threat_type: String,
|
||||
severity: String,
|
||||
src_ip: String,
|
||||
dst_ip: String,
|
||||
dst_port: Number,
|
||||
protocol: String,
|
||||
description: String,
|
||||
event_at: String,
|
||||
flow_id: { type: String, index: true },
|
||||
}, baseOptions);
|
||||
|
||||
// ─── App Categories (per agent) ───────────────────────────────────────────────
|
||||
const AppCategoryStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
category_label: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── System Events (per agent) ─────────────────────────────────────────────────
|
||||
const EventSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
event_id: Number,
|
||||
event_type: String,
|
||||
severity: String,
|
||||
description: String,
|
||||
category_label: String,
|
||||
ip_address: String,
|
||||
mac_address: String,
|
||||
event_at: Date,
|
||||
flow_id: { type: String, index: true },
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Compound Indexes for common dashboard queries ─────────────────────────────
|
||||
SummarySchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
AppStatSchema.index({ agent_uuid: 1, timestamp: -1, download: -1 });
|
||||
DeviceStatSchema.index({ agent_uuid: 1, ip_address: 1 }, { unique: true });
|
||||
FlowSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
FlowSchema.index({ agent_uuid: 1, flow_id: 1 });
|
||||
FlowSchema.index({ agent_uuid: 1, protocol: 1, timestamp: -1 });
|
||||
FlowSchema.index({ agent_uuid: 1, domain: 1, timestamp: -1 });
|
||||
FlowSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
|
||||
FlowSchema.index({ site_uuid: 1, src_mac: 1, timestamp: -1 });
|
||||
ThreatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
AppCategoryStatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
EventSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
|
||||
// ── Per-Device Per-Application Stats (synced from proxy) ─────────────────
|
||||
const DeviceAppStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
ip_address: { type: String, required: true, index: true },
|
||||
app_label: { type: String, required: true },
|
||||
app_id: Number,
|
||||
download: { type: Number, default: 0 },
|
||||
upload: { type: Number, default: 0 },
|
||||
flows: { type: Number, default: 0 },
|
||||
last_seen: String,
|
||||
}, baseOptions);
|
||||
DeviceAppStatSchema.index({ agent_uuid: 1, ip_address: 1, timestamp: -1 });
|
||||
DeviceAppStatSchema.index({ ip_address: 1, app_label: 1, timestamp: -1 });
|
||||
DeviceAppStatSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
|
||||
|
||||
const telemetrySchemas = require('./SchemasTelemetry');
|
||||
const auxSchemas = require('./SchemasAux');
|
||||
|
||||
module.exports = {
|
||||
Summary: mongoose.model('Summary', SummarySchema),
|
||||
AppStat: mongoose.model('AppStat', AppStatSchema),
|
||||
ProtocolStat: mongoose.model('ProtocolStat', ProtocolStatSchema),
|
||||
DeviceStat: mongoose.model('DeviceStat', DeviceStatSchema),
|
||||
DeviceAppStat: mongoose.model('DeviceAppStat', DeviceAppStatSchema),
|
||||
Flow: mongoose.model('Flow', FlowSchema),
|
||||
Threat: mongoose.model('Threat', ThreatSchema),
|
||||
AppCategoryStat: mongoose.model('AppCategoryStat', AppCategoryStatSchema),
|
||||
Event: mongoose.model('Event', EventSchema),
|
||||
...auxSchemas,
|
||||
...telemetrySchemas
|
||||
};
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
// backend/models/Session.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// MongoDB User Session Schema for remote revocation capability
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const mongoose = require('mongoose');
|
||||
|
||||
const SessionSchema = new mongoose.Schema({
|
||||
user_id: { type: mongoose.Schema.Types.ObjectId, ref: 'User', required: true, index: true },
|
||||
ip_address: { type: String, default: 'Unknown' },
|
||||
user_agent: { type: String, default: 'Unknown' },
|
||||
session_token: { type: String, required: true, unique: true }, // JWT JTI or unique token hash
|
||||
last_active: { type: Date, default: Date.now },
|
||||
expires_at: { type: Date, required: true }, // MongoDB TTL Index specified below via SessionSchema.index
|
||||
}, {
|
||||
timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
|
||||
});
|
||||
|
||||
// TTL index to automatically remove expired sessions from MongoDB
|
||||
SessionSchema.index({ expires_at: 1 }, { expireAfterSeconds: 0 });
|
||||
|
||||
module.exports = mongoose.model('Session', SessionSchema);
|
||||
@@ -14,11 +14,15 @@ const UserSchema = new mongoose.Schema({
|
||||
password_hash: { type: String, required: true },
|
||||
account_name: { type: String, default: null },
|
||||
profile_picture: { type: String, default: null },
|
||||
role: { type: String, enum: ['SUPER_ADMIN', 'TENANT_ADMIN', 'SOC_ANALYST', 'ENGINEER', 'AGENT_VIEWER'], default: 'AGENT_VIEWER' },
|
||||
role: { type: String, enum: ['SUPER_ADMIN', 'EXECUTIVE', 'TENANT_ADMIN', 'SOC_ANALYST', 'ENGINEER', 'AGENT_VIEWER', 'COMPANY_ADMIN', 'COMPANY_OPERATOR', 'COMPANY_VIEWER'], default: 'AGENT_VIEWER' },
|
||||
site_uuid: { type: String, default: null, index: true },
|
||||
agent_uuid: { type: String, default: null },
|
||||
company_name: { type: String, default: null, index: true },
|
||||
agent_uuids: { type: [String], default: [] },
|
||||
created_by: { type: String, default: null, index: true },
|
||||
is_active: { type: Boolean, default: true },
|
||||
login_attempts: { type: Number, default: 0 },
|
||||
lockout_until: { type: Date, default: null },
|
||||
}, {
|
||||
timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
|
||||
});
|
||||
|
||||
+2718
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,312 @@
|
||||
// backend/scheduler.js
|
||||
const cron = require('node-cron');
|
||||
const netify = require('./netify');
|
||||
const db = require('./database');
|
||||
const SITE_UUID = process.env.NETIFY_SITE_UUID || 'dummy_site_uuid';
|
||||
|
||||
let isRunning = false;
|
||||
|
||||
async function runPoll() {
|
||||
if (isRunning) {
|
||||
console.log('[Scheduler] Poll sedang berjalan, skip.');
|
||||
return;
|
||||
}
|
||||
isRunning = true;
|
||||
const fetchedAt = new Date().toISOString();
|
||||
console.log(`[Scheduler] Mulai polling... (${fetchedAt})`);
|
||||
|
||||
try {
|
||||
// 0. Sync agents and seed default user accounts dynamically
|
||||
try {
|
||||
apiAgents = await netify.fetchAgents();
|
||||
if (apiAgents && apiAgents.length > 0) {
|
||||
db.syncAgentUsers(apiAgents);
|
||||
console.log(`[Scheduler] OK Sync Agents : ${apiAgents.length} agen terdeteksi`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[Scheduler] Gagal sync agent users:', err.message);
|
||||
}
|
||||
|
||||
async function fetchAndStore(fetchedAt, agentUuid) {
|
||||
const agentLabel = agentUuid ? agentUuid : 'Global';
|
||||
console.log(`[Scheduler] Fetching data for ${agentLabel}`);
|
||||
// 1. Top Aplikasi
|
||||
const apps = await netify.fetchTopApps(1440, 20, agentUuid);
|
||||
if (apps && Array.isArray(apps)) {
|
||||
db.insertBandwidthApps(apps, fetchedAt, SITE_UUID, agentUuid);
|
||||
console.log(`[Scheduler] OK Apps : ${apps.length} baris`);
|
||||
} else {
|
||||
console.log(`[Scheduler] -- Apps : tidak ada data`);
|
||||
}
|
||||
|
||||
// 2. Top Devices — pakai fetchDiscoveredDevices yg sudah dinormalisasi
|
||||
const devices = await netify.fetchDiscoveredDevices(1440, 200, agentUuid);
|
||||
if (devices && Array.isArray(devices)) {
|
||||
db.insertDevices(devices, fetchedAt, SITE_UUID, agentUuid);
|
||||
console.log(`[Scheduler] OK Devices : ${devices.length} baris`);
|
||||
} else {
|
||||
console.log(`[Scheduler] -- Devices : tidak ada data`);
|
||||
}
|
||||
|
||||
// 3. Top Protokol
|
||||
const protocols = await netify.fetchTopProtocols(1440, 20, agentUuid);
|
||||
if (protocols && Array.isArray(protocols)) {
|
||||
db.insertProtocols(protocols, fetchedAt, SITE_UUID, agentUuid);
|
||||
console.log(`[Scheduler] OK Protocols : ${protocols.length} baris`);
|
||||
} else {
|
||||
console.log(`[Scheduler] -- Protocols : tidak ada data`);
|
||||
}
|
||||
|
||||
// 4. Top Negara
|
||||
const countries = await netify.fetchTopCountries(1440, 15, agentUuid);
|
||||
if (countries && Array.isArray(countries)) {
|
||||
db.insertCountries(countries, fetchedAt, SITE_UUID, agentUuid);
|
||||
console.log(`[Scheduler] OK Countries : ${countries.length} baris`);
|
||||
} else {
|
||||
console.log(`[Scheduler] -- Countries : tidak ada data`);
|
||||
}
|
||||
|
||||
// 5. Top Domain/DNS
|
||||
const domains = await netify.fetchTopDomains(1440, 20, agentUuid);
|
||||
if (domains && Array.isArray(domains)) {
|
||||
db.insertDNS(domains, fetchedAt, SITE_UUID, agentUuid);
|
||||
console.log(`[Scheduler] OK DNS : ${domains.length} baris`);
|
||||
} else {
|
||||
console.log(`[Scheduler] -- DNS : tidak ada data`);
|
||||
}
|
||||
|
||||
// 6. Flows — pakai local_ip sebagai proxy
|
||||
const flows = await netify.fetchFlows(200, agentUuid);
|
||||
if (flows && Array.isArray(flows)) {
|
||||
db.insertFlows(flows, fetchedAt, SITE_UUID, agentUuid);
|
||||
console.log(`[Scheduler] OK Flows : ${flows.length} baris`);
|
||||
} else {
|
||||
console.log(`[Scheduler] -- Flows : tidak ada data`);
|
||||
}
|
||||
|
||||
// 7. Threats — dari Events Status
|
||||
const threats = await netify.fetchCyberThreats(1440, 50, agentUuid);
|
||||
if (threats && Array.isArray(threats)) {
|
||||
db.insertThreats(threats, fetchedAt, SITE_UUID, agentUuid);
|
||||
console.log(`[Scheduler] OK Threats : ${threats.length} baris`);
|
||||
} else {
|
||||
console.log(`[Scheduler] -- Threats : tidak ada data`);
|
||||
}
|
||||
|
||||
// 8. Events Log
|
||||
const events = await netify.fetchEvents(50, agentUuid);
|
||||
if (events && Array.isArray(events)) {
|
||||
db.insertEvents(events, fetchedAt, SITE_UUID, agentUuid);
|
||||
console.log(`[Scheduler] OK Events : ${events.length} baris`);
|
||||
} else {
|
||||
console.log(`[Scheduler] -- Events : tidak ada data`);
|
||||
}
|
||||
|
||||
// 10. App Categories
|
||||
const appCats = await netify.fetchTopAppCategories(1440, 15, agentUuid);
|
||||
if (appCats?.length) { db.insertAppCategories(appCats, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK AppCats : ${appCats.length} baris`); }
|
||||
else console.log(`[Scheduler] -- AppCats : tidak ada data`);
|
||||
|
||||
// 11. Continents
|
||||
const continents = await netify.fetchTopContinents(1440, 10, agentUuid);
|
||||
if (continents?.length) { db.insertContinents(continents, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK Continents : ${continents.length} baris`); }
|
||||
else console.log(`[Scheduler] -- Continents : tidak ada data`);
|
||||
|
||||
// 12. Regions
|
||||
const regions = await netify.fetchTopRegions(1440, 20, agentUuid);
|
||||
if (regions?.length) { db.insertRegions(regions, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK Regions : ${regions.length} baris`); }
|
||||
else console.log(`[Scheduler] -- Regions : tidak ada data`);
|
||||
|
||||
// 13. Cities
|
||||
const cities = await netify.fetchTopCities(1440, 20, agentUuid);
|
||||
if (cities?.length) { db.insertCities(cities, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK Cities : ${cities.length} baris`); }
|
||||
else console.log(`[Scheduler] -- Cities : tidak ada data`);
|
||||
|
||||
// 14. VLANs
|
||||
const vlans = await netify.fetchTopVLANs(1440, 20, agentUuid);
|
||||
if (vlans?.length) { db.insertVLANs(vlans, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK VLANs : ${vlans.length} baris`); }
|
||||
else console.log(`[Scheduler] -- VLANs : tidak ada data`);
|
||||
|
||||
// 15. Interfaces
|
||||
const ifaces = await netify.fetchTopInterfaces(1440, 20, agentUuid);
|
||||
if (ifaces?.length) { db.insertInterfaces(ifaces, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK Interfaces : ${ifaces.length} baris`); }
|
||||
else console.log(`[Scheduler] -- Interfaces : tidak ada data`);
|
||||
|
||||
// 16. Flow Types
|
||||
const flowTypes = await netify.fetchTopFlowTypes(1440, 10, agentUuid);
|
||||
if (flowTypes?.length) { db.insertFlowTypes(flowTypes, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK FlowTypes : ${flowTypes.length} baris`); }
|
||||
else console.log(`[Scheduler] -- FlowTypes : tidak ada data`);
|
||||
|
||||
// 17. Flow Origins
|
||||
const flowOrigins = await netify.fetchTopFlowOrigins(1440, 10, agentUuid);
|
||||
if (flowOrigins?.length) { db.insertFlowOrigins(flowOrigins, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK FlowOrigin : ${flowOrigins.length} baris`); }
|
||||
else console.log(`[Scheduler] -- FlowOrigin : tidak ada data`);
|
||||
|
||||
// 18. IP Versions
|
||||
const ipVersions = await netify.fetchTopIPVersions(1440, 5, agentUuid);
|
||||
if (ipVersions?.length) { db.insertIPVersions(ipVersions, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK IPVersions : ${ipVersions.length} baris`); }
|
||||
else console.log(`[Scheduler] -- IPVersions : tidak ada data`);
|
||||
|
||||
// 19. Remote IPs
|
||||
const remoteIPs = await netify.fetchTopRemoteIPs(1440, 20, agentUuid);
|
||||
if (remoteIPs?.length) { db.insertRemoteIPs(remoteIPs, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK RemoteIPs : ${remoteIPs.length} baris`); }
|
||||
else console.log(`[Scheduler] -- RemoteIPs : tidak ada data`);
|
||||
|
||||
// 20. MAC Bandwidth
|
||||
const macBW = await netify.fetchTopLocalMACs(1440, 50, agentUuid);
|
||||
if (macBW?.length) { db.insertMACBandwidth(macBW, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK MACBandwdh : ${macBW.length} baris`); }
|
||||
else console.log(`[Scheduler] -- MACBandwdh : tidak ada data`);
|
||||
|
||||
// 9. Bandwidth Timeline
|
||||
const summary = await netify.fetchBandwidthSummary(1440, agentUuid);
|
||||
const devCount = devices?.length ?? 0;
|
||||
if (summary) {
|
||||
db.insertBandwidthTimeline({ ...summary, devices: devCount }, fetchedAt, SITE_UUID, agentUuid);
|
||||
console.log(`[Scheduler] OK Timeline : saved`);
|
||||
} else {
|
||||
console.log(`[Scheduler] -- Timeline : gagal ambil data`);
|
||||
}
|
||||
|
||||
// 21. TLS Versions
|
||||
const tlsVer = await netify.fetchTLSVersions(1440, 10, agentUuid);
|
||||
if (tlsVer?.length) { db.insertTLSVersions(tlsVer, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK TLS Ver : ${tlsVer.length} baris`); }
|
||||
else console.log(`[Scheduler] -- TLS Ver : tidak ada data`);
|
||||
|
||||
// 22. TLS Ciphers
|
||||
const tlsCipher = await netify.fetchTLSCiphers(1440, 15, agentUuid);
|
||||
if (tlsCipher?.length) { db.insertTLSCiphers(tlsCipher, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK TLS Cipher : ${tlsCipher.length} baris`); }
|
||||
else console.log(`[Scheduler] -- TLS Cipher : tidak ada data`);
|
||||
|
||||
// 23. TLS Security
|
||||
const tlsSec = await netify.fetchTLSSecurity(1440, 10, agentUuid);
|
||||
if (tlsSec?.length) { db.insertTLSSecurity(tlsSec, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK TLS Sec : ${tlsSec.length} baris`); }
|
||||
else console.log(`[Scheduler] -- TLS Sec : tidak ada data`);
|
||||
|
||||
// 24. NetBIOS Hostnames
|
||||
const netbios = await netify.fetchNetBIOSHostnames(1440, 30, agentUuid);
|
||||
if (netbios?.length) { db.insertNetBIOSHostnames(netbios, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK NetBIOS : ${netbios.length} baris`); }
|
||||
else console.log(`[Scheduler] -- NetBIOS : tidak ada data`);
|
||||
|
||||
// 25. Discovery OS (standalone — OS yang terdeteksi di jaringan)
|
||||
const discOs = await netify.fetchTopDiscoveryOS(1440, 20, agentUuid);
|
||||
if (discOs?.length) { db.insertDiscoveryOS(discOs, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK DiscOS : ${discOs.length} baris`); }
|
||||
else console.log(`[Scheduler] -- DiscOS : tidak ada data`);
|
||||
|
||||
// 26. DHCP Class Fingerprint
|
||||
const dhcpFp = await netify.fetchDHCPClassFingerprints(1440, 30, agentUuid);
|
||||
if (dhcpFp?.length) { db.insertDHCPFingerprints(dhcpFp, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK DHCP FP : ${dhcpFp.length} baris`); }
|
||||
else console.log(`[Scheduler] -- DHCP FP : tidak ada data`);
|
||||
|
||||
// 27. HTTP User-Agent
|
||||
const userAgents = await netify.fetchHTTPUserAgents(1440, 30, agentUuid);
|
||||
if (userAgents?.length) { db.insertHTTPUserAgents(userAgents, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK UserAgent : ${userAgents.length} baris`); }
|
||||
else console.log(`[Scheduler] -- UserAgent : tidak ada data`);
|
||||
|
||||
// 28. HTTPS SNI Hostname
|
||||
const sniHosts = await netify.fetchSNIHostnames(1440, 30, agentUuid);
|
||||
if (sniHosts?.length) { db.insertSNIHostnames(sniHosts, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK SNI Host : ${sniHosts.length} baris`); }
|
||||
else console.log(`[Scheduler] -- SNI Host : tidak ada data`);
|
||||
|
||||
// 29. SSL Server Common Name
|
||||
const sslCN = await netify.fetchSSLServerCN(1440, 30, agentUuid);
|
||||
if (sslCN?.length) { db.insertSSLServerCN(sslCN, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK SSL CN : ${sslCN.length} baris`); }
|
||||
else console.log(`[Scheduler] -- SSL CN : tidak ada data`);
|
||||
|
||||
// 30. QUIC Hostname
|
||||
const quicHosts = await netify.fetchQUICHostnames(1440, 30, agentUuid);
|
||||
if (quicHosts?.length) { db.insertQUICHostnames(quicHosts, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK QUIC Host : ${quicHosts.length} baris`); }
|
||||
else console.log(`[Scheduler] -- QUIC Host : tidak ada data`);
|
||||
|
||||
// 31. BitTorrent Info Hash
|
||||
const btHashes = await netify.fetchBitTorrentInfoHashes(1440, 30, agentUuid);
|
||||
if (btHashes?.length) { db.insertBitTorrentHashes(btHashes, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK BT Hash : ${btHashes.length} baris`); }
|
||||
else console.log(`[Scheduler] -- BT Hash : tidak ada data`);
|
||||
|
||||
// 32. SSH Client (field: ssh_client)
|
||||
const sshClient = await netify.fetchSSHClients(1440, 20, agentUuid);
|
||||
if (sshClient?.length) { db.insertSSHVersions(sshClient, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK SSH Client : ${sshClient.length} baris`); }
|
||||
else console.log(`[Scheduler] -- SSH Client : tidak ada data`);
|
||||
|
||||
// 32b. SSH Server (field: ssh_server)
|
||||
const sshServer = await netify.fetchSSHServers(1440, 20, agentUuid);
|
||||
if (sshServer?.length) { db.insertSSHVersions(sshServer, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK SSH Server : ${sshServer.length} baris`); }
|
||||
else console.log(`[Scheduler] -- SSH Server : tidak ada data`);
|
||||
|
||||
// 33. mDNS Hostname (Chromecast, Apple TV, etc.)
|
||||
const mdnsHosts = await netify.fetchMDNSHostnames(1440, 30, agentUuid);
|
||||
if (mdnsHosts?.length) { db.insertMDNSHostnames(mdnsHosts, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK mDNS Host : ${mdnsHosts.length} baris`); }
|
||||
else console.log(`[Scheduler] -- mDNS Host : tidak ada data`);
|
||||
|
||||
// ─── INTELLIGENCE 22-30 (derive dari data yang tersedia) ─────────────────
|
||||
|
||||
// 34. Cryptocurrency Mining (derive dari apps + flows ke port mining)
|
||||
const cryptoMining = await netify.fetchCryptoMining(50, agentUuid);
|
||||
if (cryptoMining?.length) { db.insertCryptoMining(cryptoMining, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK CryptoMine : ${cryptoMining.length} baris`); }
|
||||
else console.log(`[Scheduler] -- CryptoMine : tidak ada data`);
|
||||
|
||||
// 35. Device Discovery (derive dari flows + bandwidth per-IP)
|
||||
const devDisc = await netify.fetchDeviceDiscovery(100, agentUuid);
|
||||
if (devDisc?.length) { db.insertDeviceDiscovery(devDisc, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK DevDisc : ${devDisc.length} baris`); }
|
||||
else console.log(`[Scheduler] -- DevDisc : tidak ada data`);
|
||||
|
||||
// 36. Encryption Audit (derive dari flows per-IP: port encrypted vs plain)
|
||||
const encAudit = await netify.fetchEncryptionAudit(50, agentUuid);
|
||||
if (encAudit?.length) { db.insertEncryptionAudit(encAudit, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK EncAudit : ${encAudit.length} baris`); }
|
||||
else console.log(`[Scheduler] -- EncAudit : tidak ada data`);
|
||||
|
||||
// 37. Insecure Protocols (derive dari top protocols)
|
||||
const insecProto = await netify.fetchInsecureProtocols(1440, 50, agentUuid);
|
||||
if (insecProto?.length) { db.insertInsecureProtocols(insecProto, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK InsecProto : ${insecProto.length} baris`); }
|
||||
else console.log(`[Scheduler] -- InsecProto : tidak ada data`);
|
||||
|
||||
// 38. IP Reputation (derive dari top remote_ip + high-risk countries)
|
||||
const ipRep = await netify.fetchIPReputation(50, agentUuid);
|
||||
if (ipRep?.length) { db.insertIPReputation(ipRep, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK IPRepute : ${ipRep.length} baris`); }
|
||||
else console.log(`[Scheduler] -- IPRepute : tidak ada data`);
|
||||
|
||||
// 39. Server Discovery (derive dari flows ke port server well-known)
|
||||
const srvDisc = await netify.fetchServerDiscovery(100, agentUuid);
|
||||
if (srvDisc?.length) { db.insertServerDiscovery(srvDisc, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK SrvDisc : ${srvDisc.length} baris`); }
|
||||
else console.log(`[Scheduler] -- SrvDisc : tidak ada data`);
|
||||
|
||||
// 40. Tor Detection (derive dari apps/hostnames mengandung "tor")
|
||||
const torDet = await netify.fetchTorDetection(50, agentUuid);
|
||||
if (torDet?.length) { db.insertTorDetection(torDet, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK TorDet : ${torDet.length} baris`); }
|
||||
else console.log(`[Scheduler] -- TorDet : tidak ada data`);
|
||||
|
||||
// 41. Unencrypted Password (derive dari flows ke port cleartext auth)
|
||||
const unencPwd = await netify.fetchUnencryptedPasswords(50, agentUuid);
|
||||
if (unencPwd?.length) { db.insertUnencryptedPasswords(unencPwd, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK UnencPwd : ${unencPwd.length} baris`); }
|
||||
else console.log(`[Scheduler] -- UnencPwd : tidak ada data`);
|
||||
|
||||
// 42. VPN Detection (derive dari apps/protocols/ports VPN)
|
||||
const vpnDet = await netify.fetchVPNDetection(50, agentUuid);
|
||||
if (vpnDet?.length) { db.insertVPNDetection(vpnDet, fetchedAt, SITE_UUID, agentUuid); console.log(`[Scheduler] OK VPNDet : ${vpnDet.length} baris`); }
|
||||
else console.log(`[Scheduler] -- VPNDet : tidak ada data`);
|
||||
|
||||
}
|
||||
|
||||
// --- Main loop
|
||||
await fetchAndStore(fetchedAt, null);
|
||||
if (apiAgents && apiAgents.length > 0) {
|
||||
for (const agent of apiAgents) {
|
||||
if (agent && agent.uuid) {
|
||||
await fetchAndStore(fetchedAt, agent.uuid);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[Scheduler] ERROR:', err);
|
||||
} finally {
|
||||
isRunning = false;
|
||||
console.log(`[Scheduler] Poll selesai.\n`);
|
||||
}
|
||||
}
|
||||
|
||||
function startScheduler() {
|
||||
runPoll();
|
||||
cron.schedule('* * * * *', () => runPoll());
|
||||
console.log('[Scheduler] Aktif. Polling setiap 1 menit.\n');
|
||||
}
|
||||
|
||||
module.exports = { startScheduler, runPoll };
|
||||
+163
-123
@@ -1,123 +1,163 @@
|
||||
// backend/server.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// BackOne Backend API Server
|
||||
//
|
||||
// Tanggung jawab backend ini adalah READ-ONLY dari MongoDB.
|
||||
// Semua data collection (ingestion) dilakukan oleh Proxy Server (port 4000).
|
||||
// Backend TIDAK memanggil DPI API secara langsung.
|
||||
//
|
||||
// Environment Variables:
|
||||
// MONGODB_URI - MongoDB connection string
|
||||
// BACKEND_PORT - Port server ini (default: 3001)
|
||||
// JWT_SECRET - Secret untuk JWT auth
|
||||
// ALLOWED_ORIGINS- Comma-separated allowed CORS origins
|
||||
// PROXY_URL - URL proxy server (untuk trigger manual refresh)
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
|
||||
|
||||
const express = require('express');
|
||||
const cors = require('cors');
|
||||
const cookieParser = require('cookie-parser');
|
||||
const jwt = require('jsonwebtoken');
|
||||
const connectDB = require('./db/mongoose');
|
||||
|
||||
// ─── Connect to MongoDB (read-only mode) ──────────────────────────────────────
|
||||
connectDB();
|
||||
|
||||
const app = express();
|
||||
const PORT = process.env.BACKEND_PORT || 3001;
|
||||
|
||||
// ─── Middleware ────────────────────────────────────────────────────────────────
|
||||
const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS
|
||||
? process.env.ALLOWED_ORIGINS.split(',')
|
||||
: ['http://localhost:3000', 'http://127.0.0.1:3000'];
|
||||
|
||||
app.use(cors({
|
||||
origin: (origin, callback) => {
|
||||
if (!origin) return callback(null, true);
|
||||
if (ALLOWED_ORIGINS.includes(origin)) {
|
||||
callback(null, true);
|
||||
} else {
|
||||
callback(new Error('Blocked by CORS policy (Unauthorized Origin)'));
|
||||
}
|
||||
},
|
||||
credentials: true
|
||||
}));
|
||||
app.use(express.json());
|
||||
app.use(cookieParser());
|
||||
|
||||
// ─── Public Routes ────────────────────────────────────────────────────────────
|
||||
const authRoutes = require('./routes/auth');
|
||||
const { getUploadsDir } = require('./routes/auth/helpers');
|
||||
app.use('/api/auth', authRoutes);
|
||||
app.use('/api/uploads', express.static(getUploadsDir()));
|
||||
|
||||
// ─── Auth Middleware ──────────────────────────────────────────────────────────
|
||||
const { requireAuth } = require('./middleware/auth');
|
||||
const { getTimeFilter, getBaseFilter } = require('./routes/dashboard/helpers');
|
||||
const {
|
||||
generateMacFromIp,
|
||||
resolveVendorFromIp,
|
||||
resolveDeviceTypeFromIp,
|
||||
resolveOSFromIp,
|
||||
generateAutoLabel
|
||||
} = require('./deviceResolver');
|
||||
|
||||
// ─── Protected Dashboard Routes ───────────────────────────────────────────────
|
||||
const dashboardRoutes = require('./routes/dashboard');
|
||||
|
||||
// Override /api/dashboard/app-details to show real-time device mapping per application
|
||||
app.get('/api/dashboard/app-details', requireAuth, (req, res) => {
|
||||
require('./routes/appDetailsHandler')(req, res, {
|
||||
getTimeFilter,
|
||||
getBaseFilter
|
||||
});
|
||||
});
|
||||
|
||||
// Override /api/dashboard/device-details to map real-time classifications (Facebook, YouTube, etc.)
|
||||
app.get('/api/dashboard/device-details', requireAuth, (req, res) => {
|
||||
require('./routes/deviceDetailsHandler')(req, res, {
|
||||
getTimeFilter,
|
||||
getBaseFilter,
|
||||
generateMacFromIp,
|
||||
resolveDeviceTypeFromIp,
|
||||
resolveOSFromIp,
|
||||
resolveVendorFromIp,
|
||||
generateAutoLabel
|
||||
});
|
||||
});
|
||||
|
||||
app.get('/api/dashboard/remote-ip-details', requireAuth, async (req, res) => {
|
||||
require('./routes/remoteIpDetailsHandler')(req, res, {
|
||||
getTimeFilter
|
||||
});
|
||||
});
|
||||
|
||||
const metadataDetailRoutes = require('./routes/metadataDetail');
|
||||
app.use('/api/dashboard/metadata-detail', requireAuth, metadataDetailRoutes);
|
||||
|
||||
const categoryDetailRoutes = require('./routes/categoryDetail');
|
||||
app.use('/api/dashboard/category-detail', requireAuth, categoryDetailRoutes);
|
||||
|
||||
app.use('/api/dashboard', requireAuth, dashboardRoutes);
|
||||
|
||||
|
||||
|
||||
|
||||
// ─── Health Check ─────────────────────────────────────────────────────────────
|
||||
app.get('/api/health', (req, res) => {
|
||||
res.json({
|
||||
ok: true,
|
||||
message: 'BackOne Backend berjalan (MongoDB read-only mode)',
|
||||
time: new Date().toISOString()
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Start Server ─────────────────────────────────────────────────────────────
|
||||
app.listen(PORT, () => {
|
||||
console.log(`\n🚀 BackOne API Server berjalan di http://localhost:${PORT}`);
|
||||
console.log(`🔌 API Health : http://localhost:${PORT}/api/health`);
|
||||
console.log(`📡 Mode : READ-ONLY dari MongoDB (data dikirim oleh Proxy Server)\n`);
|
||||
});
|
||||
// backend/server.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Polyfill global crypto for Node 18 compatibility (required by mongodb driver)
|
||||
if (typeof globalThis.crypto === 'undefined') {
|
||||
globalThis.crypto = require('crypto');
|
||||
}
|
||||
|
||||
// BackOne Backend API Server
|
||||
//
|
||||
// Tanggung jawab backend ini adalah READ-ONLY dari MongoDB.
|
||||
// Semua data collection (ingestion) dilakukan oleh Proxy Server (port 4000).
|
||||
// Backend TIDAK memanggil DPI API secara langsung.
|
||||
//
|
||||
// Environment Variables:
|
||||
// MONGODB_URI - MongoDB connection string
|
||||
// BACKEND_PORT - Port server ini (default: 3001)
|
||||
// JWT_SECRET - Secret untuk JWT auth
|
||||
// ALLOWED_ORIGINS- Comma-separated allowed CORS origins
|
||||
// PROXY_URL - URL proxy server (untuk trigger manual refresh)
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const path = require('path');
|
||||
const envFile = process.env.NODE_ENV === 'production' ? '.env.production' : '.env.local';
|
||||
require('dotenv').config({ path: path.join(__dirname, '..', envFile) });
|
||||
|
||||
const express = require('express');
|
||||
const cors = require('cors');
|
||||
const cookieParser = require('cookie-parser');
|
||||
const jwt = require('jsonwebtoken');
|
||||
const connectDB = require('./db/mongoose');
|
||||
|
||||
// ─── Connect to MongoDB (read-only mode) ──────────────────────────────────────
|
||||
connectDB();
|
||||
|
||||
const app = express();
|
||||
const PORT = process.env.BACKEND_PORT || 3001;
|
||||
|
||||
// ─── Middleware ────────────────────────────────────────────────────────────────
|
||||
const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS
|
||||
? process.env.ALLOWED_ORIGINS.split(',')
|
||||
: ['http://localhost:3000', 'http://127.0.0.1:3000'];
|
||||
|
||||
app.use(cors({
|
||||
origin: (origin, callback) => {
|
||||
if (!origin) return callback(null, true);
|
||||
if (ALLOWED_ORIGINS.includes(origin)) {
|
||||
callback(null, true);
|
||||
} else {
|
||||
callback(new Error('Blocked by CORS policy (Unauthorized Origin)'));
|
||||
}
|
||||
},
|
||||
credentials: true
|
||||
}));
|
||||
app.use(express.json({ limit: '10mb' }));
|
||||
app.use(express.urlencoded({ extended: true, limit: '10mb' }));
|
||||
app.use(cookieParser());
|
||||
|
||||
app.use((req, res, next) => {
|
||||
if (req.originalUrl && req.originalUrl.includes('/api/dashboard')) {
|
||||
try {
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const logPath = path.join(__dirname, '../scratch/http_requests.log');
|
||||
const logLine = `[${new Date().toISOString()}] ${req.method} ${req.originalUrl} - Query: ${JSON.stringify(req.query)}\n`;
|
||||
fs.appendFileSync(logPath, logLine);
|
||||
} catch (e) {
|
||||
console.error('Logger error:', e.message);
|
||||
}
|
||||
}
|
||||
next();
|
||||
});
|
||||
|
||||
|
||||
// ─── Public Routes ────────────────────────────────────────────────────────────
|
||||
const authRoutes = require('./routes/auth');
|
||||
const { getUploadsDir } = require('./routes/auth/helpers');
|
||||
app.use('/api/auth', authRoutes);
|
||||
app.use('/api/uploads', express.static(getUploadsDir()));
|
||||
|
||||
// ─── Auth Middleware ──────────────────────────────────────────────────────────
|
||||
const { requireAuth } = require('./middleware/auth');
|
||||
const { getTimeFilter, getBaseFilter } = require('./routes/dashboard/helpers');
|
||||
const {
|
||||
generateMacFromIp,
|
||||
resolveVendorFromIp,
|
||||
resolveDeviceTypeFromIp,
|
||||
resolveOSFromIp,
|
||||
generateAutoLabel
|
||||
} = require('./deviceResolver');
|
||||
|
||||
// ─── Protected Dashboard Routes ───────────────────────────────────────────────
|
||||
const dashboardRoutes = require('./routes/dashboard');
|
||||
|
||||
// Override /api/dashboard/app-details to show real-time device mapping per application
|
||||
app.get('/api/dashboard/app-details', requireAuth, (req, res) => {
|
||||
require('./routes/appDetailsHandler')(req, res, {
|
||||
getTimeFilter,
|
||||
getBaseFilter
|
||||
});
|
||||
});
|
||||
|
||||
// Override /api/dashboard/device-details to map real-time classifications (Facebook, YouTube, etc.)
|
||||
app.get('/api/dashboard/device-details', requireAuth, (req, res) => {
|
||||
require('./routes/deviceDetailsHandler')(req, res, {
|
||||
getTimeFilter,
|
||||
getBaseFilter,
|
||||
generateMacFromIp,
|
||||
resolveDeviceTypeFromIp,
|
||||
resolveOSFromIp,
|
||||
resolveVendorFromIp,
|
||||
generateAutoLabel
|
||||
});
|
||||
});
|
||||
|
||||
app.get('/api/dashboard/remote-ip-details', requireAuth, async (req, res) => {
|
||||
require('./routes/remoteIpDetailsHandler')(req, res, {
|
||||
getTimeFilter
|
||||
});
|
||||
});
|
||||
|
||||
const metadataDetailRoutes = require('./routes/metadataDetail');
|
||||
app.use('/api/dashboard/metadata-detail', requireAuth, metadataDetailRoutes);
|
||||
|
||||
const categoryDetailRoutes = require('./routes/categoryDetail');
|
||||
app.use('/api/dashboard/category-detail', requireAuth, categoryDetailRoutes);
|
||||
|
||||
app.use('/api/dashboard', requireAuth, dashboardRoutes);
|
||||
|
||||
|
||||
|
||||
|
||||
// ─── Health Check ─────────────────────────────────────────────────────────────
|
||||
app.get('/api/health', (req, res) => {
|
||||
res.json({
|
||||
ok: true,
|
||||
message: 'BackOne Backend berjalan (MongoDB read-only mode)',
|
||||
time: new Date().toISOString()
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Global JSON Error Handler ────────────────────────────────────────────────
|
||||
// Menangkap semua error yang tidak di-handle (termasuk multer, mongoose, dll.)
|
||||
// dan memastikan response selalu JSON, BUKAN HTML default Express.
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
app.use((err, req, res, next) => {
|
||||
console.error('[Global Error Handler]', err.message || err);
|
||||
const status = err.status || err.statusCode || 500;
|
||||
res.status(status).json({
|
||||
error: err.message || 'Internal server error',
|
||||
code: err.code || undefined,
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Start Server ─────────────────────────────────────────────────────────────
|
||||
// Bind to 127.0.0.1 in production to prevent direct external access to port 3001.
|
||||
// All external traffic must go through the reverse proxy (Apache/Nginx) at port 80/443.
|
||||
const BIND_HOST = process.env.NODE_ENV === 'production' ? '127.0.0.1' : '0.0.0.0';
|
||||
app.listen(PORT, BIND_HOST, () => {
|
||||
console.log(`\n🚀 BackOne API Server berjalan di http://${BIND_HOST}:${PORT}`);
|
||||
console.log(`🔌 API Health : http://${BIND_HOST}:${PORT}/api/health`);
|
||||
console.log(`📡 Mode : READ-ONLY dari MongoDB (data dikirim oleh Proxy Server)`);
|
||||
console.log(`🔒 Security : Bound to ${BIND_HOST} (internal only in production)\n`);
|
||||
});
|
||||
@@ -0,0 +1,135 @@
|
||||
const cron = require('node-cron');
|
||||
const netify = require('../netify');
|
||||
const { Summary, AppStat, ProtocolStat, DeviceStat, Flow, Threat } = require('../models/Schemas');
|
||||
|
||||
const SITE_UUID = process.env.NETIFY_SITE_UUID || process.env.BACKONE_SITE_UUID;
|
||||
let isRunning = false;
|
||||
|
||||
async function runPoll() {
|
||||
if (isRunning) return;
|
||||
isRunning = true;
|
||||
const timestamp = new Date();
|
||||
console.log(`[Mongo-Ingestion] Started polling at ${timestamp.toISOString()}`);
|
||||
|
||||
try {
|
||||
const agents = await netify.fetchAgents();
|
||||
const agentList = agents && agents.length > 0 ? agents.map(a => a.uuid) : [null]; // null for global
|
||||
|
||||
for (const agentUuid of agentList) {
|
||||
console.log(`[Mongo-Ingestion] Fetching data for Agent: ${agentUuid || 'Global'}`);
|
||||
|
||||
// 1. Summary
|
||||
const summary = await netify.fetchBandwidthSummary(1440, agentUuid);
|
||||
if (summary) {
|
||||
await new Summary({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
...summary
|
||||
}).save();
|
||||
}
|
||||
|
||||
// 2. Apps
|
||||
const apps = await netify.fetchTopApps(1440, 200, agentUuid); // high limit for data lake
|
||||
if (apps && apps.length > 0) {
|
||||
const appDocs = apps.map(app => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
app_label: app.application?.label || 'Unknown',
|
||||
download: app.download || 0,
|
||||
upload: app.upload || 0,
|
||||
flows: app.flows || 0
|
||||
}));
|
||||
await AppStat.insertMany(appDocs);
|
||||
}
|
||||
|
||||
const devices = await netify.fetchDiscoveredDevices(1440, 500, agentUuid);
|
||||
if (devices && devices.length > 0) {
|
||||
const devDocs = devices.map(d => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
ip_address: d.ip_address,
|
||||
mac_address: d.mac_address,
|
||||
device_label: d.device_label,
|
||||
device_type: d.device_type,
|
||||
os_label: d.os_label,
|
||||
manufacturer: d.manufacturer,
|
||||
download: d.download || 0,
|
||||
upload: d.upload || 0,
|
||||
flows: d.flows || 0,
|
||||
last_seen: d.last_seen
|
||||
})).filter(d => d.ip_address); // Ensure ip_address exists to avoid validation error
|
||||
if (devDocs.length > 0) {
|
||||
await DeviceStat.insertMany(devDocs);
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Flows
|
||||
const flows = await netify.fetchFlows(500, agentUuid);
|
||||
if (flows && flows.length > 0) {
|
||||
const flowDocs = flows.map(f => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
flow_id: f.flow_id,
|
||||
src_ip: f.src_ip,
|
||||
src_mac: f.src_mac,
|
||||
dst_ip: f.dst_ip,
|
||||
dst_port: f.dst_port,
|
||||
protocol: f.protocol,
|
||||
app_label: f.app_label,
|
||||
domain: f.domain,
|
||||
download: f.download || 0,
|
||||
upload: f.upload || 0,
|
||||
first_seen: f.first_seen,
|
||||
last_seen: f.last_seen
|
||||
})).filter(f => f.src_ip);
|
||||
if (flowDocs.length > 0) {
|
||||
await Flow.insertMany(flowDocs);
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Threats
|
||||
const threats = await netify.fetchCyberThreats(agentUuid);
|
||||
if (threats && threats.length > 0) {
|
||||
const threatDocs = threats.map(t => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
threat_type: t.threat_type || 'Unknown Threat',
|
||||
severity: t.severity || 'Medium',
|
||||
src_ip: t.src_ip,
|
||||
dst_ip: t.dst_ip,
|
||||
dst_port: t.dst_port,
|
||||
protocol: t.protocol,
|
||||
description: t.description,
|
||||
event_at: t.event_at || new Date().toISOString()
|
||||
}));
|
||||
if (threatDocs.length > 0) {
|
||||
await Threat.insertMany(threatDocs);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[Mongo-Ingestion] Error during polling:', error);
|
||||
} finally {
|
||||
isRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
function startScheduler() {
|
||||
// Run every 5 minutes
|
||||
cron.schedule('*/5 * * * *', () => {
|
||||
runPoll();
|
||||
});
|
||||
console.log('[Mongo-Ingestion] Scheduler started (every 5 minutes)');
|
||||
|
||||
// Initial run
|
||||
runPoll();
|
||||
}
|
||||
|
||||
module.exports = { startScheduler };
|
||||
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
const http = require('http');
|
||||
|
||||
http.get('http://localhost:3001/api/dashboard/tls-versions', {
|
||||
headers: {
|
||||
'Cookie': 'token=test', // Just checking schema, if it requires auth we might need to mock or use the proxy
|
||||
}
|
||||
}, (res) => {
|
||||
let data = '';
|
||||
res.on('data', chunk => data += chunk);
|
||||
res.on('end', () => {
|
||||
console.log("Response TLS Versions:");
|
||||
console.log(data.slice(0, 500));
|
||||
});
|
||||
});
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
@@ -0,0 +1,60 @@
|
||||
// check-mongo.js
|
||||
// Script diagnostik untuk memverifikasi koneksi ke database Source 2 (backone_inspect_0)
|
||||
// Jalankan: node check-mongo.js
|
||||
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.join(__dirname, '.env.local') });
|
||||
|
||||
const mongoose = require('mongoose');
|
||||
|
||||
const MONGODB_URI = process.env.MONGODB_URI;
|
||||
|
||||
if (!MONGODB_URI) {
|
||||
console.error('[ERROR] MONGODB_URI tidak ditemukan di .env.local');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log('\n╔════════════════════════════════════════════════╗');
|
||||
console.log('║ Source 2 — MongoDB Connection Diagnostic ║');
|
||||
console.log('╚════════════════════════════════════════════════╝\n');
|
||||
console.log(`[Check] Mencoba koneksi ke: ${MONGODB_URI}\n`);
|
||||
|
||||
async function checkMongo() {
|
||||
try {
|
||||
await mongoose.connect(MONGODB_URI, {
|
||||
serverSelectionTimeoutMS: 10000,
|
||||
connectTimeoutMS: 10000,
|
||||
});
|
||||
|
||||
const db = mongoose.connection.db;
|
||||
const dbName = db.databaseName;
|
||||
|
||||
console.log(`[OK] Berhasil terhubung ke MongoDB!`);
|
||||
console.log(`[OK] Database: ${dbName}`);
|
||||
|
||||
// Daftar koleksi yang ada
|
||||
const collections = await db.listCollections().toArray();
|
||||
if (collections.length === 0) {
|
||||
console.log('[INFO] Database masih kosong — belum ada koleksi.');
|
||||
} else {
|
||||
console.log(`[INFO] Koleksi yang ada (${collections.length}):`);
|
||||
for (const col of collections) {
|
||||
const count = await db.collection(col.name).countDocuments();
|
||||
console.log(` - ${col.name}: ${count} dokumen`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n[RESULT] ✅ STEP 8 PASS — Koneksi ke database Source 2 berhasil.\n');
|
||||
process.exit(0);
|
||||
} catch (err) {
|
||||
console.error(`[ERROR] Gagal terhubung ke MongoDB: ${err.message}`);
|
||||
console.error('\nPossible causes:');
|
||||
console.error(' 1. Host "mongodb-netify" tidak bisa dijangkau (butuh VPN/SSH tunnel)');
|
||||
console.error(' 2. Kredensial backone_inspect:backone_inspect salah');
|
||||
console.error(' 3. MongoDB belum berjalan di server tujuan');
|
||||
console.error('\n[RESULT] ❌ STEP 8 FAIL — Hubungi atasan untuk verifikasi koneksi.\n');
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
checkMongo();
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# deploy-server-setup.sh
|
||||
# Script yang dijalankan di server setelah file di-upload
|
||||
# Path: /home/adminbackend/web/dev.demoplace.my.id/public_html/
|
||||
# =============================================================================
|
||||
|
||||
set -e
|
||||
DEPLOY_DIR="/home/adminbackend/web/dev.demoplace.my.id/public_html"
|
||||
cd "$DEPLOY_DIR"
|
||||
|
||||
echo "=== [1/6] Checking environment ==="
|
||||
node --version
|
||||
npm --version
|
||||
pm2 --version || npm install -g pm2
|
||||
|
||||
echo ""
|
||||
echo "=== [2/6] Installing backend dependencies ==="
|
||||
cd "$DEPLOY_DIR/backend"
|
||||
npm install --omit=dev --legacy-peer-deps
|
||||
cd "$DEPLOY_DIR"
|
||||
|
||||
echo ""
|
||||
echo "=== [3/6] Installing proxy dependencies ==="
|
||||
cd "$DEPLOY_DIR/proxy"
|
||||
npm install --omit=dev --legacy-peer-deps
|
||||
cd "$DEPLOY_DIR"
|
||||
|
||||
echo ""
|
||||
echo "=== [4/6] Creating required directories ==="
|
||||
mkdir -p logs
|
||||
mkdir -p scratch
|
||||
|
||||
echo ""
|
||||
echo "=== [5/6] Stopping old PM2 processes (if any) ==="
|
||||
pm2 delete source2-proxy 2>/dev/null || echo "source2-proxy: not running"
|
||||
pm2 delete source2-backend 2>/dev/null || echo "source2-backend: not running"
|
||||
pm2 delete source2-frontend 2>/dev/null || echo "source2-frontend: not running"
|
||||
|
||||
echo ""
|
||||
echo "=== [6/6] Starting PM2 processes ==="
|
||||
pm2 start ecosystem.config.js --env production
|
||||
pm2 save
|
||||
pm2 list
|
||||
|
||||
echo ""
|
||||
echo "=== DEPLOY COMPLETE ==="
|
||||
echo "Frontend : http://127.0.0.1:3010"
|
||||
echo "Backend : http://127.0.0.1:3011"
|
||||
echo "Proxy : http://127.0.0.1:4010"
|
||||
echo ""
|
||||
echo "Check logs with: pm2 logs source2-backend --lines 30"
|
||||
+1
-4
@@ -53,12 +53,9 @@ services:
|
||||
- MONGODB_URI=mongodb://mongodb:27017/backone_dpi
|
||||
- PROXY_PORT=4000
|
||||
# Mode 1: kumpulkan SEMUA agent (default)
|
||||
# Ubah ke PROXY_COLLECT_MODE=agent dan isi PROXY_AGENT_UUID untuk mode spesifik (1 agent)
|
||||
# Ubah ke PROXY_COLLECT_MODE=agents dan isi PROXY_AGENT_UUIDS untuk mode multi-agent
|
||||
# Ubah ke PROXY_COLLECT_MODE=agent dan isi PROXY_AGENT_UUID untuk mode spesifik
|
||||
- PROXY_COLLECT_MODE=${PROXY_COLLECT_MODE:-all}
|
||||
- PROXY_AGENT_UUID=${PROXY_AGENT_UUID:-}
|
||||
- PROXY_AGENT_UUIDS=${PROXY_AGENT_UUIDS:-}
|
||||
- PROXY_AGENT_DELAY_MS=${PROXY_AGENT_DELAY_MS:-5000}
|
||||
- PROXY_CRON_SCHEDULE=${PROXY_CRON_SCHEDULE:-*/5 * * * *}
|
||||
networks:
|
||||
- backone-infra
|
||||
|
||||
+55
-13
@@ -1,22 +1,64 @@
|
||||
// ecosystem.config.js — PM2 Configuration for Source 2 (dev.demoplace.my.id)
|
||||
module.exports = {
|
||||
apps: [
|
||||
{
|
||||
name: "backone-proxy",
|
||||
script: "./proxy/index.js",
|
||||
env_file: ".env.production"
|
||||
name: 'source2-proxy',
|
||||
script: './proxy/index.js',
|
||||
cwd: '/home/adminbackend/web/dev.demoplace.my.id/public_html',
|
||||
instances: 1,
|
||||
exec_mode: 'fork',
|
||||
watch: false,
|
||||
node_args: '--max-old-space-size=1024',
|
||||
max_memory_restart: '1200M',
|
||||
restart_delay: 5000,
|
||||
max_restarts: 10,
|
||||
env_file: '.env.production',
|
||||
env: { NODE_ENV: 'production' },
|
||||
error_file: './logs/proxy-error.log',
|
||||
out_file: './logs/proxy-out.log',
|
||||
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
|
||||
merge_logs: true,
|
||||
},
|
||||
{
|
||||
name: "backone-backend",
|
||||
script: "./backend/server.js",
|
||||
env_file: ".env.production"
|
||||
name: 'source2-backend',
|
||||
script: './backend/server.js',
|
||||
cwd: '/home/adminbackend/web/dev.demoplace.my.id/public_html',
|
||||
instances: 1,
|
||||
exec_mode: 'fork',
|
||||
watch: false,
|
||||
node_args: '--max-old-space-size=256',
|
||||
max_memory_restart: '400M',
|
||||
restart_delay: 3000,
|
||||
max_restarts: 10,
|
||||
env_file: '.env.production',
|
||||
env: { NODE_ENV: 'production' },
|
||||
error_file: './logs/backend-error.log',
|
||||
out_file: './logs/backend-out.log',
|
||||
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
|
||||
merge_logs: true,
|
||||
},
|
||||
{
|
||||
name: "backone-frontend",
|
||||
script: "server.js",
|
||||
env_file: ".env.production",
|
||||
name: 'source2-frontend',
|
||||
script: 'start-with-env.js',
|
||||
cwd: '/home/adminbackend/web/dev.demoplace.my.id/public_html',
|
||||
instances: 1,
|
||||
exec_mode: 'fork',
|
||||
watch: false,
|
||||
node_args: '--max-old-space-size=512',
|
||||
max_memory_restart: '700M',
|
||||
restart_delay: 3000,
|
||||
max_restarts: 10,
|
||||
env_file: '.env.production',
|
||||
env: {
|
||||
PORT: 8009
|
||||
}
|
||||
}
|
||||
]
|
||||
NODE_ENV: 'production',
|
||||
PORT: 3010,
|
||||
HOSTNAME: '127.0.0.1',
|
||||
NEXT_TELEMETRY_DISABLED: '1',
|
||||
},
|
||||
error_file: './logs/frontend-error.log',
|
||||
out_file: './logs/frontend-out.log',
|
||||
log_date_format: 'YYYY-MM-DD HH:mm:ss Z',
|
||||
merge_logs: true,
|
||||
},
|
||||
],
|
||||
};
|
||||
+21
-21
@@ -1,21 +1,21 @@
|
||||
import { defineConfig, globalIgnores } from "eslint/config";
|
||||
import nextVitals from "eslint-config-next/core-web-vitals";
|
||||
import nextTs from "eslint-config-next/typescript";
|
||||
|
||||
const eslintConfig = defineConfig([
|
||||
...nextVitals,
|
||||
...nextTs,
|
||||
// Override default ignores of eslint-config-next.
|
||||
globalIgnores([
|
||||
// Default ignores of eslint-config-next:
|
||||
".next/**",
|
||||
"out/**",
|
||||
"build/**",
|
||||
"next-env.d.ts",
|
||||
"backend/**",
|
||||
"proxy/**",
|
||||
"test/**",
|
||||
]),
|
||||
]);
|
||||
|
||||
export default eslintConfig;
|
||||
import { defineConfig, globalIgnores } from "eslint/config";
|
||||
import nextVitals from "eslint-config-next/core-web-vitals";
|
||||
import nextTs from "eslint-config-next/typescript";
|
||||
|
||||
const eslintConfig = defineConfig([
|
||||
...nextVitals,
|
||||
...nextTs,
|
||||
// Override default ignores of eslint-config-next.
|
||||
globalIgnores([
|
||||
// Default ignores of eslint-config-next:
|
||||
".next/**",
|
||||
"out/**",
|
||||
"build/**",
|
||||
"next-env.d.ts",
|
||||
"backend/**",
|
||||
"proxy/**",
|
||||
"test/**",
|
||||
]),
|
||||
]);
|
||||
|
||||
export default eslintConfig;
|
||||
+259
@@ -0,0 +1,259 @@
|
||||
/**
|
||||
* git-push-iso.js
|
||||
* Push ke Gitea & GitHub menggunakan isomorphic-git (pure JS, tanpa system git)
|
||||
*
|
||||
* CARA KERJA:
|
||||
* 1. Clone dari Gitea (untuk dapat history 75 commits)
|
||||
* 2. Salin file proyek terbaru ke folder clone
|
||||
* 3. Commit perubahan
|
||||
* 4. Push ke Gitea
|
||||
* 5. Push ke GitHub dengan remote tambahan
|
||||
*/
|
||||
|
||||
const git = require('isomorphic-git');
|
||||
const http = require('isomorphic-git/http/node');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const os = require('os');
|
||||
|
||||
// ─── CONFIG ────────────────────────────────────────────────────────────────
|
||||
const PROJECT_DIR = path.resolve(__dirname);
|
||||
|
||||
// Gitea
|
||||
const GITEA_URL = 'https://git.proit.id/rafif/Deep-Package-Inspection';
|
||||
const GITEA_BRANCH = 'Proxy_Server_API_backone.cloud';
|
||||
const GITEA_USER = 'rafif';
|
||||
const GITEA_PASS = 'NetWorking.0';
|
||||
|
||||
// GitHub
|
||||
const GITHUB_URL = 'https://github.com/Rafif-Riqullah-Siregar/BackOne-Deep-Package-Inspection';
|
||||
const GITHUB_BRANCH = 'Proxy-Server-API-backone.cloud';
|
||||
// GitHub token (diisi nanti, atau bisa kosong dulu untuk test)
|
||||
const GITHUB_TOKEN = process.env.GITHUB_TOKEN || '';
|
||||
|
||||
// Commit message
|
||||
const COMMIT_MSG = `feat(source2): push all latest files - device labeling, help system, proxy docs, database isolation fix
|
||||
|
||||
- Added DOKUMENTASI-FILTER-PER-SITE.md (site isolation docs)
|
||||
- Fixed start-with-env.js to force-load .env.production
|
||||
- Fixed MONGODB_URI hostname from mongodb-netify to mongodb.prod.proit.id
|
||||
- Updated .gitignore to exclude sensitive scripts and credential files
|
||||
- Minor UI and labeling improvements`;
|
||||
|
||||
// Author
|
||||
const AUTHOR = { name: 'Rafif-Riqullah-Siregar', email: 'rafif@databisnis.id' };
|
||||
|
||||
// ─── GITIGNORE PATTERNS ──────────────────────────────────────────────────────
|
||||
// File/folder yang TIDAK boleh di-push (dari .gitignore)
|
||||
const EXCLUDED_PATTERNS = [
|
||||
'node_modules',
|
||||
'.next',
|
||||
'.env',
|
||||
'.env.local',
|
||||
'.env.production',
|
||||
'.env.development',
|
||||
'coverage',
|
||||
'build',
|
||||
'out',
|
||||
'.DS_Store',
|
||||
'*.log',
|
||||
'*.pem',
|
||||
'.vercel',
|
||||
'*.tsbuildinfo',
|
||||
'next-env.d.ts',
|
||||
'*.db', '*.db-shm', '*.db-wal', '*.sqlite',
|
||||
'Laporan_*.docx',
|
||||
'temp_docx',
|
||||
'*.zip',
|
||||
'AGENTS.md', 'CLAUDE.md', '.agents',
|
||||
'docs', 'plans',
|
||||
'temp.json', 'scratch',
|
||||
// Sensitive scripts
|
||||
'compare-netify-vs-dashboard.js',
|
||||
'ssh-read-source1-proxy.js',
|
||||
'check-frontend-uri-now.js',
|
||||
'verify-final.js',
|
||||
'check-frontend-uri.js',
|
||||
'ssh-check-logs.js',
|
||||
// Sensitive docs
|
||||
'BUKTI-AKSES-MONGODB.txt',
|
||||
'DOKUMENTASI-PROXY-NETIFY.md',
|
||||
];
|
||||
|
||||
function shouldExclude(filePath) {
|
||||
const parts = filePath.split(/[/\\]/);
|
||||
for (const pattern of EXCLUDED_PATTERNS) {
|
||||
for (const part of parts) {
|
||||
if (pattern.startsWith('*')) {
|
||||
const ext = pattern.slice(1);
|
||||
if (part.endsWith(ext)) return true;
|
||||
} else if (part === pattern || filePath.includes(pattern)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async function getGitFiles(dir, baseDir = dir) {
|
||||
const files = [];
|
||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
const relPath = path.relative(baseDir, fullPath).replace(/\\/g, '/');
|
||||
if (shouldExclude(relPath) || entry.name === '.git') continue;
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...await getGitFiles(fullPath, baseDir));
|
||||
} else {
|
||||
files.push(relPath);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('╔══════════════════════════════════════════════════════════════╗');
|
||||
console.log('║ GIT PUSH (isomorphic-git) → Gitea + GitHub ║');
|
||||
console.log('╚══════════════════════════════════════════════════════════════╝\n');
|
||||
|
||||
// ─── STEP 1: Clone dari Gitea ke temp dir ──────────────────────────────────
|
||||
const tmpDir = path.join(os.tmpdir(), `dpi-push-${Date.now()}`);
|
||||
console.log(`[1] Cloning dari Gitea ke temp: ${tmpDir}`);
|
||||
fs.mkdirSync(tmpDir, { recursive: true });
|
||||
|
||||
try {
|
||||
await git.clone({
|
||||
fs, http,
|
||||
dir: tmpDir,
|
||||
url: GITEA_URL,
|
||||
ref: GITEA_BRANCH,
|
||||
singleBranch: true,
|
||||
depth: 10, // ambil 10 commit terakhir saja (cukup untuk push)
|
||||
onAuth: () => ({ username: GITEA_USER, password: GITEA_PASS }),
|
||||
onProgress: ({ phase, loaded, total }) => {
|
||||
if (total) process.stdout.write(`\r ${phase}: ${loaded}/${total} `);
|
||||
},
|
||||
});
|
||||
console.log(`\n ✅ Clone berhasil dari Gitea branch ${GITEA_BRANCH}`);
|
||||
} catch (err) {
|
||||
console.error(`\n ❌ Clone dari Gitea gagal: ${err.message}`);
|
||||
console.log(' → Coba dengan username tanpa domain (tanpa @databisnis.id)');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// ─── STEP 2: Salin file project terbaru ke temp dir ────────────────────────
|
||||
console.log(`\n[2] Menyalin file terbaru dari project ke clone...`);
|
||||
const projectFiles = await getGitFiles(PROJECT_DIR);
|
||||
let copied = 0;
|
||||
for (const relPath of projectFiles) {
|
||||
const src = path.join(PROJECT_DIR, relPath);
|
||||
const dst = path.join(tmpDir, relPath);
|
||||
fs.mkdirSync(path.dirname(dst), { recursive: true });
|
||||
fs.copyFileSync(src, dst);
|
||||
copied++;
|
||||
}
|
||||
console.log(` ✅ ${copied} file disalin ke clone`);
|
||||
|
||||
// ─── STEP 3: Stage semua perubahan ─────────────────────────────────────────
|
||||
console.log(`\n[3] Staging semua perubahan...`);
|
||||
const statusMatrix = await git.statusMatrix({ fs, dir: tmpDir });
|
||||
let staged = 0;
|
||||
for (const [filepath, head, workdir, stage] of statusMatrix) {
|
||||
if (workdir !== stage) {
|
||||
if (workdir === 0) {
|
||||
// File dihapus
|
||||
await git.remove({ fs, dir: tmpDir, filepath });
|
||||
} else {
|
||||
// File baru atau dimodifikasi
|
||||
await git.add({ fs, dir: tmpDir, filepath });
|
||||
}
|
||||
staged++;
|
||||
}
|
||||
}
|
||||
console.log(` ✅ ${staged} file di-stage`);
|
||||
|
||||
if (staged === 0) {
|
||||
console.log(' ℹ️ Tidak ada perubahan yang perlu di-commit!');
|
||||
return cleanup(tmpDir);
|
||||
}
|
||||
|
||||
// ─── STEP 4: Commit ─────────────────────────────────────────────────────────
|
||||
console.log(`\n[4] Membuat commit...`);
|
||||
const sha = await git.commit({
|
||||
fs,
|
||||
dir: tmpDir,
|
||||
author: AUTHOR,
|
||||
committer: AUTHOR,
|
||||
message: COMMIT_MSG,
|
||||
});
|
||||
console.log(` ✅ Commit dibuat: ${sha.slice(0, 8)}`);
|
||||
|
||||
// ─── STEP 5: Push ke Gitea ──────────────────────────────────────────────────
|
||||
console.log(`\n[5] Push ke Gitea (${GITEA_URL})...`);
|
||||
try {
|
||||
await git.push({
|
||||
fs, http,
|
||||
dir: tmpDir,
|
||||
remote: 'origin',
|
||||
ref: GITEA_BRANCH,
|
||||
onAuth: () => ({ username: GITEA_USER, password: GITEA_PASS }),
|
||||
onProgress: ({ phase, loaded, total }) => {
|
||||
if (total) process.stdout.write(`\r ${phase}: ${loaded}/${total} `);
|
||||
},
|
||||
});
|
||||
console.log(`\n ✅ Push ke Gitea BERHASIL! Branch: ${GITEA_BRANCH}`);
|
||||
} catch (err) {
|
||||
console.error(`\n ❌ Push ke Gitea gagal: ${err.message}`);
|
||||
}
|
||||
|
||||
// ─── STEP 6: Push ke GitHub ─────────────────────────────────────────────────
|
||||
console.log(`\n[6] Push ke GitHub (${GITHUB_URL})...`);
|
||||
|
||||
// Tambah remote GitHub
|
||||
const remotes = await git.listRemotes({ fs, dir: tmpDir });
|
||||
const hasGithub = remotes.some(r => r.remote === 'github');
|
||||
if (!hasGithub) {
|
||||
await git.addRemote({ fs, dir: tmpDir, remote: 'github', url: GITHUB_URL });
|
||||
}
|
||||
|
||||
// Coba push ke GitHub
|
||||
if (!GITHUB_TOKEN) {
|
||||
console.log(' ⚠️ GITHUB_TOKEN tidak di-set. GitHub push membutuhkan Personal Access Token.');
|
||||
console.log(' → Set environment variable: $env:GITHUB_TOKEN = "ghp_XXXX"');
|
||||
console.log(' → Lalu jalankan: node git-push-iso.js');
|
||||
} else {
|
||||
try {
|
||||
await git.push({
|
||||
fs, http,
|
||||
dir: tmpDir,
|
||||
remote: 'github',
|
||||
ref: GITHUB_BRANCH,
|
||||
remoteRef: GITHUB_BRANCH,
|
||||
onAuth: () => ({ username: 'Rafif-Riqullah-Siregar', password: GITHUB_TOKEN }),
|
||||
onProgress: ({ phase, loaded, total }) => {
|
||||
if (total) process.stdout.write(`\r ${phase}: ${loaded}/${total} `);
|
||||
},
|
||||
});
|
||||
console.log(`\n ✅ Push ke GitHub BERHASIL! Branch: ${GITHUB_BRANCH}`);
|
||||
} catch (err) {
|
||||
console.error(`\n ❌ Push ke GitHub gagal: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
cleanup(tmpDir);
|
||||
console.log('\n╔══════════════════════════════════════════════════════════════╗');
|
||||
console.log('║ SELESAI ║');
|
||||
console.log('╚══════════════════════════════════════════════════════════════╝');
|
||||
}
|
||||
|
||||
function cleanup(tmpDir) {
|
||||
try {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
console.log(`\n[cleanup] Temp dir dihapus: ${tmpDir}`);
|
||||
} catch(e) {}
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('\n[FATAL]', err.message);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,35 @@
|
||||
[
|
||||
{
|
||||
"_id": "6a584fdb36539224fa4cbfd9",
|
||||
"agent_uuid": "F6-2V-DT-8A",
|
||||
"site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
|
||||
"latitude": -6.2263304,
|
||||
"longitude": 106.4247322,
|
||||
"label": "CPI Balaraja Agent Office",
|
||||
"created_at": "2026-07-14T04:03:09.294Z",
|
||||
"updated_at": "2026-07-14T04:03:09.294Z",
|
||||
"__v": 0
|
||||
},
|
||||
{
|
||||
"_id": "6a584fdb36539224fa4cbfda",
|
||||
"agent_uuid": "2F-TF-1D-GK",
|
||||
"site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
|
||||
"latitude": -6.3763318,
|
||||
"longitude": 106.8983017,
|
||||
"label": "JRP Cibubur Agent",
|
||||
"created_at": "2026-07-14T04:03:09.303Z",
|
||||
"updated_at": "2026-07-14T04:57:22.288Z",
|
||||
"__v": 0
|
||||
},
|
||||
{
|
||||
"_id": "6a584fdb36539224fa4cbfdb",
|
||||
"agent_uuid": "8A-V3-PB-85",
|
||||
"site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
|
||||
"latitude": -6.2253265,
|
||||
"longitude": 106.8061484,
|
||||
"label": "IFG LT.18 Agent HQ",
|
||||
"created_at": "2026-07-14T04:03:09.322Z",
|
||||
"updated_at": "2026-07-14T04:03:09.322Z",
|
||||
"__v": 0
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,35 @@
|
||||
[
|
||||
{
|
||||
"_id": "6a584fdb36539224fa4cbfd6",
|
||||
"site_uuid": "default",
|
||||
"brand_name": "BackOne",
|
||||
"brand_logo": "/backone-logo.png",
|
||||
"footer_copyright": "PT. Data Bisnis Solusi",
|
||||
"primary_color": "#E11D48",
|
||||
"created_at": "2026-07-14T02:15:21.201Z",
|
||||
"updated_at": "2026-07-27T01:03:28.716Z",
|
||||
"__v": 0
|
||||
},
|
||||
{
|
||||
"_id": "6a584fdb36539224fa4cbfd7",
|
||||
"site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
|
||||
"brand_name": "SIAB",
|
||||
"brand_logo": "/siab-logo.png",
|
||||
"footer_copyright": "PT. Data Bisnis Solusi",
|
||||
"primary_color": "#3B82F6",
|
||||
"created_at": "2026-07-14T02:15:21.204Z",
|
||||
"updated_at": "2026-07-27T01:03:28.796Z",
|
||||
"__v": 0
|
||||
},
|
||||
{
|
||||
"_id": "6a584fdb36539224fa4cbfd8",
|
||||
"site_uuid": "d7902405_0dc2_458b_8584_ed4d24b64f24",
|
||||
"brand_name": "Nexus",
|
||||
"brand_logo": "/nexus-logo.png",
|
||||
"footer_copyright": "PT. Nexus Solusi",
|
||||
"primary_color": "#8B5CF6",
|
||||
"created_at": "2026-07-14T02:15:21.206Z",
|
||||
"updated_at": "2026-07-27T01:03:28.799Z",
|
||||
"__v": 0
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,217 @@
|
||||
[
|
||||
{
|
||||
"_id": "6a509b014fa14ba76d96ed33",
|
||||
"username": "admin",
|
||||
"password_hash": "$2a$10$uaBO91aVN9kwWS3rhCBvmu3uV00QFzMjorwqPK/AkhsGKKaLoFJoG",
|
||||
"account_name": "BackOne Administrator",
|
||||
"role": "SUPER_ADMIN",
|
||||
"site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
|
||||
"is_active": true,
|
||||
"created_at": "2026-07-08T06:20:27.502Z",
|
||||
"updated_at": "2026-07-21T06:57:52.516Z",
|
||||
"profile_picture": "profile-1784254528937-270868858.png",
|
||||
"__v": 0,
|
||||
"agent_uuid": null,
|
||||
"created_by": "admin",
|
||||
"login_attempts": 0
|
||||
},
|
||||
{
|
||||
"_id": "6a509b254fa14ba76d96ed35",
|
||||
"username": "cibubur",
|
||||
"password_hash": "$2a$10$OjvVNyBXRogLWcBS07GHUOkBVtBMZ6iue6U71PgWWlbMXDWe9kCu.",
|
||||
"account_name": "JRP Cibubur Agent",
|
||||
"role": "AGENT_VIEWER",
|
||||
"site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
|
||||
"agent_uuid": "2F-TF-1D-GK",
|
||||
"is_active": true,
|
||||
"created_at": "2026-07-14T03:39:34.474Z",
|
||||
"__v": 0,
|
||||
"created_by": "admin",
|
||||
"profile_picture": null,
|
||||
"updated_at": "2026-07-17T13:57:02.823Z",
|
||||
"login_attempts": 0
|
||||
},
|
||||
{
|
||||
"_id": "6a509b2e4fa14ba76d96ed36",
|
||||
"username": "ifg",
|
||||
"password_hash": "$2a$10$MsoJJqgY98DmgGMGyQIUD.PRA5kdbpXWhvNHFRakeipuJGF2F/73q",
|
||||
"account_name": "IFG LT.18 Agent",
|
||||
"role": "AGENT_VIEWER",
|
||||
"site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
|
||||
"agent_uuid": "8A-V3-PB-85",
|
||||
"is_active": true,
|
||||
"created_at": "2026-07-14T03:39:52.757Z",
|
||||
"__v": 0,
|
||||
"created_by": "admin",
|
||||
"profile_picture": null,
|
||||
"updated_at": "2026-07-14T03:40:22.272Z"
|
||||
},
|
||||
{
|
||||
"_id": "6a509b394fa14ba76d96ed37",
|
||||
"username": "balaraja",
|
||||
"password_hash": "$2a$10$sx7XNdylDOr1XCy4PjjEuOrCoIWhayMNYwNlsAjspHVnmrz0xmQ9a",
|
||||
"account_name": "CPI Balaraja Agent",
|
||||
"role": "AGENT_VIEWER",
|
||||
"site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
|
||||
"agent_uuid": "F6-2V-DT-8A",
|
||||
"is_active": true,
|
||||
"created_at": "2026-07-14T03:40:14.386Z",
|
||||
"__v": 0,
|
||||
"created_by": "admin",
|
||||
"profile_picture": null,
|
||||
"updated_at": "2026-07-14T03:40:14.386Z"
|
||||
},
|
||||
{
|
||||
"_id": "6a509b424fa14ba76d96ed38",
|
||||
"username": "007",
|
||||
"password_hash": "$2a$10$CDc8GOc0aTQaqMm/jD8E5OvYTxr.lbTPdrRbC6O1D2MDRzClbgyA6",
|
||||
"account_name": "Gateway 007 Agent",
|
||||
"role": "AGENT_VIEWER",
|
||||
"site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
|
||||
"agent_uuid": "YW-6I-61-LL",
|
||||
"is_active": true,
|
||||
"created_at": "2026-07-14T03:40:51.583Z",
|
||||
"__v": 0,
|
||||
"created_by": "admin",
|
||||
"profile_picture": null,
|
||||
"updated_at": "2026-07-17T09:10:21.716Z",
|
||||
"login_attempts": 3,
|
||||
"lockout_until": "2026-07-17T09:25:21.716Z"
|
||||
},
|
||||
{
|
||||
"_id": "6a54b314301004e28818f8e2",
|
||||
"username": "bsd",
|
||||
"password_hash": "$2a$10$IIZtN8coQVLfptktA0bO2eIzaCpy3yIGtr9EUWsSf9MdTUaztx8eW",
|
||||
"account_name": "Fazza BSD",
|
||||
"profile_picture": null,
|
||||
"role": "AGENT_VIEWER",
|
||||
"site_uuid": "d7902405_0dc2_458b_8584_ed4d24b64f24",
|
||||
"agent_uuid": "1T-5Q-RC-AS",
|
||||
"is_active": true,
|
||||
"created_at": "2026-07-14T03:38:04.913Z",
|
||||
"updated_at": "2026-07-14T03:38:04.913Z",
|
||||
"__v": 0,
|
||||
"created_by": "admin"
|
||||
},
|
||||
{
|
||||
"_id": "6a54b332301004e28818f8e8",
|
||||
"username": "jkt",
|
||||
"password_hash": "$2a$10$EE0G6vQ6qbN6MYj2BSjqWOdJN2q/LmBcYRLZ578higbfLjnOzRKuW",
|
||||
"account_name": "Fazza JKT",
|
||||
"profile_picture": null,
|
||||
"role": "AGENT_VIEWER",
|
||||
"site_uuid": "d7902405_0dc2_458b_8584_ed4d24b64f24",
|
||||
"agent_uuid": "2N-ID-VQ-AL",
|
||||
"is_active": true,
|
||||
"created_at": "2026-07-14T03:38:25.721Z",
|
||||
"updated_at": "2026-07-14T03:38:25.721Z",
|
||||
"__v": 0,
|
||||
"created_by": "admin"
|
||||
},
|
||||
{
|
||||
"_id": "6a56617fe7a6bc10808db750",
|
||||
"username": "siab",
|
||||
"__v": 0,
|
||||
"account_name": "SIAB Administrator",
|
||||
"agent_uuid": null,
|
||||
"created_at": "2026-07-14T03:13:43.107Z",
|
||||
"created_by": "admin",
|
||||
"is_active": true,
|
||||
"password_hash": "$2a$10$lGP.s16GBImnBWKlFCg9Ge7d0k0vwwhFGrlfExRs6KlhO/fW6yJE.",
|
||||
"profile_picture": "profile-1784254601947-954448750.png",
|
||||
"role": "TENANT_ADMIN",
|
||||
"site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
|
||||
"updated_at": "2026-07-17T02:16:41.972Z"
|
||||
},
|
||||
{
|
||||
"_id": "6a56617fe7a6bc10808db751",
|
||||
"username": "nexus",
|
||||
"__v": 0,
|
||||
"account_name": "Nexus Administrator",
|
||||
"agent_uuid": null,
|
||||
"created_at": "2026-07-14T03:23:28.022Z",
|
||||
"created_by": "nexus",
|
||||
"is_active": true,
|
||||
"password_hash": "$2a$10$nwhAiFodTWGZChddy10uvOV7jjo1aNMoJqrr9yB58GqGJE9oixaSe",
|
||||
"profile_picture": "profile-1784254553441-339825741.png",
|
||||
"role": "TENANT_ADMIN",
|
||||
"site_uuid": "d7902405_0dc2_458b_8584_ed4d24b64f24",
|
||||
"updated_at": "2026-07-17T09:37:28.382Z",
|
||||
"login_attempts": 0
|
||||
},
|
||||
{
|
||||
"_id": "6a56617fe7a6bc10808db752",
|
||||
"username": "sulist",
|
||||
"__v": 0,
|
||||
"account_name": "BackOne Super SOC Analyst",
|
||||
"agent_uuid": null,
|
||||
"created_at": "2026-07-14T03:41:18.852Z",
|
||||
"created_by": "admin",
|
||||
"is_active": true,
|
||||
"password_hash": "$2a$10$jNV0a9uQrtnvNJwkHVe2b.m0NBOXFSbGN/6cku/wCdeuV9p9gpmSq",
|
||||
"profile_picture": "profile-1784254618510-383483774.png",
|
||||
"role": "SOC_ANALYST",
|
||||
"site_uuid": null,
|
||||
"updated_at": "2026-07-17T02:16:58.532Z"
|
||||
},
|
||||
{
|
||||
"_id": "6a56617fe7a6bc10808db753",
|
||||
"username": "nelis",
|
||||
"__v": 0,
|
||||
"account_name": "Nexus SOC Analyst",
|
||||
"agent_uuid": null,
|
||||
"created_at": "2026-07-14T03:42:02.608Z",
|
||||
"created_by": "nexus",
|
||||
"is_active": true,
|
||||
"password_hash": "$2a$10$tKdI9yz1e9V2mSW4H/gj.udLtAtSrHJy0QxMbq6hN3mym5XxJpH.W",
|
||||
"profile_picture": "profile-1784254567483-97901195.png",
|
||||
"role": "SOC_ANALYST",
|
||||
"site_uuid": "d7902405_0dc2_458b_8584_ed4d24b64f24",
|
||||
"updated_at": "2026-07-17T02:16:07.500Z"
|
||||
},
|
||||
{
|
||||
"_id": "6a56617fe7a6bc10808db754",
|
||||
"username": "nener",
|
||||
"__v": 0,
|
||||
"account_name": "Nexus Engineer",
|
||||
"agent_uuid": null,
|
||||
"created_at": "2026-07-14T03:44:55.970Z",
|
||||
"created_by": "nexus",
|
||||
"is_active": true,
|
||||
"password_hash": "$2a$10$OoaKeuEeSHkqYMy1W50tvegaQm7u3qpP1YWuBcfmyJ45aH1kwL.Oq",
|
||||
"profile_picture": "profile-1784254581808-854860134.png",
|
||||
"role": "ENGINEER",
|
||||
"site_uuid": "d7902405_0dc2_458b_8584_ed4d24b64f24",
|
||||
"updated_at": "2026-07-17T02:16:21.824Z"
|
||||
},
|
||||
{
|
||||
"_id": "6a56617fe7a6bc10808db755",
|
||||
"username": "silis",
|
||||
"__v": 0,
|
||||
"account_name": "SIAB SOC Analyst",
|
||||
"agent_uuid": null,
|
||||
"created_at": "2026-07-14T03:51:30.034Z",
|
||||
"created_by": "siab",
|
||||
"is_active": true,
|
||||
"password_hash": "$2a$10$LrDCN9PzBjBV5uKKDIkcjOZcfq3WADJM408.VBrwlQmeqO/PbZtoG",
|
||||
"profile_picture": "profile-1784254634906-830642874.png",
|
||||
"role": "SOC_ANALYST",
|
||||
"site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
|
||||
"updated_at": "2026-07-17T02:17:14.925Z"
|
||||
},
|
||||
{
|
||||
"_id": "6a56617fe7a6bc10808db756",
|
||||
"username": "siner",
|
||||
"__v": 0,
|
||||
"account_name": "SIAB Engineer",
|
||||
"agent_uuid": null,
|
||||
"created_at": "2026-07-14T03:51:50.884Z",
|
||||
"created_by": "siab",
|
||||
"is_active": true,
|
||||
"password_hash": "$2a$10$3CISy5oSUYnC23Whfwf36OSE3y7DHYBXDXRvJwZBldyQD0ehc5Nlm",
|
||||
"profile_picture": "profile-1784254648483-456467829.png",
|
||||
"role": "ENGINEER",
|
||||
"site_uuid": "6681452d_9cae_4ff4_8ae8_0d504774265e",
|
||||
"updated_at": "2026-07-17T02:17:28.503Z"
|
||||
}
|
||||
]
|
||||
+2
-3
@@ -2,17 +2,16 @@ import type { NextConfig } from "next";
|
||||
|
||||
const nextConfig: NextConfig = {
|
||||
output: "standalone",
|
||||
serverExternalPackages: ["mongoose"],
|
||||
experimental: {
|
||||
serverActions: {
|
||||
allowedOrigins: ["demoplace.my.id", "www.demoplace.my.id"],
|
||||
allowedOrigins: ["dev.demoplace.my.id", "www.dev.demoplace.my.id"],
|
||||
},
|
||||
},
|
||||
async rewrites() {
|
||||
return [
|
||||
{
|
||||
source: '/api/:path*',
|
||||
destination: `${process.env.NEXT_PUBLIC_API_URL || 'http://127.0.0.1:3001'}/api/:path*`,
|
||||
destination: `${process.env.NEXT_PUBLIC_API_URL || 'http://127.0.0.1:3011'}/api/:path*`,
|
||||
},
|
||||
];
|
||||
},
|
||||
|
||||
@@ -1,8 +1,14 @@
|
||||
limit_req_zone $binary_remote_addr zone=api:10m rate=30r/s;
|
||||
limit_req_zone $binary_remote_addr zone=uploads:10m rate=5r/s;
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name demoplace.my.id;
|
||||
|
||||
server_tokens off; # Hide NGINX version
|
||||
server_name demoplace.my.id www.demoplace.my.id;
|
||||
|
||||
server_tokens off;
|
||||
|
||||
# Allow large file uploads for profile pictures (max 10MB)
|
||||
client_max_body_size 10m;
|
||||
|
||||
# Security Headers
|
||||
add_header X-Frame-Options "SAMEORIGIN" always;
|
||||
@@ -10,12 +16,12 @@ server {
|
||||
add_header X-Content-Type-Options "nosniff" always;
|
||||
add_header Referrer-Policy "no-referrer-when-downgrade" always;
|
||||
add_header Content-Security-Policy "default-src 'self' http: https: data: blob: 'unsafe-inline' 'unsafe-eval';" always;
|
||||
add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always;
|
||||
|
||||
# Rate Limiting zone configuration should be in nginx.conf (http block), but we can configure basic protection
|
||||
# We will pass everything to the frontend container
|
||||
|
||||
location / {
|
||||
proxy_pass http://frontend:3000;
|
||||
# Rate limiting on API endpoints
|
||||
location /api/auth/ {
|
||||
limit_req zone=api burst=20 nodelay;
|
||||
proxy_pass http://127.0.0.1:3000;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection 'upgrade';
|
||||
@@ -24,8 +30,37 @@ server {
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_hide_header X-Powered-By;
|
||||
proxy_read_timeout 30s;
|
||||
}
|
||||
|
||||
# Hide internal technologies from being sent back to the client
|
||||
# Profile picture uploads — rate limited more strictly
|
||||
location /api/auth/upload-profile-picture {
|
||||
limit_req zone=uploads burst=5 nodelay;
|
||||
client_max_body_size 10m;
|
||||
proxy_pass http://127.0.0.1:3000;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_read_timeout 60s;
|
||||
proxy_hide_header X-Powered-By;
|
||||
}
|
||||
|
||||
# All other traffic goes to Next.js frontend
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:3000;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection 'upgrade';
|
||||
proxy_set_header Host $host;
|
||||
proxy_cache_bypass $http_upgrade;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_hide_header X-Powered-By;
|
||||
proxy_read_timeout 30s;
|
||||
proxy_connect_timeout 10s;
|
||||
}
|
||||
}
|
||||
Generated
+713
-24
File diff suppressed because it is too large.
Load diff
+15
-6
@@ -1,13 +1,17 @@
|
||||
{
|
||||
"name": "netify-app",
|
||||
"version": "0.1.0",
|
||||
"name": "backone-dpi",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
},
|
||||
"scripts": {
|
||||
"dev": "concurrently --names \"NEXT,BACKEND,PROXY\" --prefix-colors \"cyan,green,yellow\" \"next dev\" \"node backend/server.js\" \"node proxy/index.js\"",
|
||||
"dev:next": "next dev",
|
||||
"dev": "concurrently --names \"MONGO,NEXT,BACKEND,PROXY\" --prefix-colors \"magenta,cyan,green,yellow\" \"node scripts/start-mongo.js\" \"next dev -p 3010\" \"node backend/server.js\" \"node proxy/index.js\"",
|
||||
"dev:central": "concurrently --names \"NEXT,BACKEND\" --prefix-colors \"cyan,green\" \"next dev -p 3010\" \"node backend/server.js\"",
|
||||
"dev:next": "next dev -p 3010",
|
||||
"dev:backend": "node backend/server.js",
|
||||
"dev:proxy": "node proxy/index.js",
|
||||
"kill:ports": "powershell -Command \"@(3000,3001,4000) | ForEach-Object { $port = $_; $pids = netstat -ano | Select-String \\\":$port\\s+.+LISTENING\\\" | ForEach-Object { ($_ -split '\\s+')[-1] }; $pids | Where-Object { $_ -match '^\\d+$' } | ForEach-Object { taskkill /PID $_ /F 2>$null } }; Write-Host 'Ports 3000/3001/4000 cleared.'\"",
|
||||
"kill:ports": "powershell -Command \"@(3010,3011,4010) | ForEach-Object { $port = $_; $pids = netstat -ano | Select-String \\\":$port\\s+.+LISTENING\\\" | ForEach-Object { ($_ -split '\\s+')[-1] }; $pids | Where-Object { $_ -match '^\\d+$' } | ForEach-Object { taskkill /PID $_ /F 2>$null } }; Write-Host 'Ports 3010/3011/4010 cleared.'\"",
|
||||
"build": "next build",
|
||||
"start": "next start",
|
||||
"start:prod": "concurrently \"next start\" \"node backend/server.js\" \"node proxy/index.js\"",
|
||||
@@ -15,9 +19,12 @@
|
||||
"backend": "node backend/server.js",
|
||||
"proxy": "node proxy/index.js",
|
||||
"proxy:bun": "bun proxy/index.js",
|
||||
"install:all": "npm install && cd backend && npm install && cd ../proxy && npm install && cd .."
|
||||
"install:all": "npm install && cd backend && npm install && cd ../proxy && npm install && cd ..",
|
||||
"deploy": "npm run build && node scripts/deploy-sftp.js",
|
||||
"deploy:sftp": "node scripts/deploy-sftp.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"@react-pdf/renderer": "^4.5.1",
|
||||
"@types/leaflet": "^1.9.21",
|
||||
"axios": "^1.18.1",
|
||||
"bcryptjs": "^3.0.3",
|
||||
@@ -31,6 +38,7 @@
|
||||
"dotenv": "^17.4.2",
|
||||
"express": "^5.2.1",
|
||||
"framer-motion": "^12.42.2",
|
||||
"isomorphic-git": "^1.40.0",
|
||||
"jsonwebtoken": "^9.0.3",
|
||||
"leaflet": "^1.9.4",
|
||||
"lucide-react": "^1.21.0",
|
||||
@@ -40,6 +48,7 @@
|
||||
"next": "16.2.9",
|
||||
"next-themes": "^0.4.6",
|
||||
"node-cron": "^4.6.0",
|
||||
"node-fetch": "^3.3.2",
|
||||
"react": "19.2.4",
|
||||
"react-dom": "19.2.4",
|
||||
"react-globe.gl": "^2.38.0",
|
||||
|
||||
+7
-7
@@ -1,7 +1,7 @@
|
||||
const config = {
|
||||
plugins: {
|
||||
"@tailwindcss/postcss": {},
|
||||
},
|
||||
};
|
||||
|
||||
export default config;
|
||||
const config = {
|
||||
plugins: {
|
||||
"@tailwindcss/postcss": {},
|
||||
},
|
||||
};
|
||||
|
||||
export default config;
|
||||
+20
-20
@@ -1,20 +1,20 @@
|
||||
FROM oven/bun:1-alpine
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install dependencies first (layer caching)
|
||||
# bun install is compatible with npm package.json / package-lock.json
|
||||
COPY package*.json ./
|
||||
RUN bun install --production
|
||||
|
||||
# Copy application source
|
||||
COPY . .
|
||||
|
||||
# Expose proxy REST API port
|
||||
EXPOSE 4000
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \
|
||||
CMD bun -e "require('http').get('http://localhost:4000/health', r => r.statusCode === 200 ? process.exit(0) : process.exit(1)).on('error', () => process.exit(1))"
|
||||
|
||||
CMD ["bun", "run", "index.js"]
|
||||
FROM oven/bun:1-alpine
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install dependencies first (layer caching)
|
||||
# bun install is compatible with npm package.json / package-lock.json
|
||||
COPY package*.json ./
|
||||
RUN bun install --production
|
||||
|
||||
# Copy application source
|
||||
COPY . .
|
||||
|
||||
# Expose proxy REST API port
|
||||
EXPOSE 4000
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \
|
||||
CMD bun -e "require('http').get('http://localhost:4000/health', r => r.statusCode === 200 ? process.exit(0) : process.exit(1)).on('error', () => process.exit(1))"
|
||||
|
||||
CMD ["bun", "run", "index.js"]
|
||||
+108
-108
@@ -1,108 +1,108 @@
|
||||
# BackOne DPI Proxy — Deployment Reference
|
||||
|
||||
Standalone Docker image for collecting Netify DPI data and writing to MongoDB.
|
||||
|
||||
---
|
||||
|
||||
## Environment Variables
|
||||
|
||||
### Required
|
||||
|
||||
| Variable | Description |
|
||||
|---|---|
|
||||
| `NETIFY_SITE_UUIDS` | Comma-separated Netify site UUIDs (or single `NETIFY_SITE_UUID`) |
|
||||
| `NETIFY_TOKEN` | Netify JWT token (or `NETIFY_JWT_TOKEN` / `NETIFY_API_KEY`) |
|
||||
|
||||
### MongoDB
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `MONGODB_URI` | `mongodb://127.0.0.1:27017/backone_dpi` | MongoDB connection string |
|
||||
|
||||
### Collection Mode
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `PROXY_COLLECT_MODE` | `all` | `all` = all agents, `agent` = single agent, `agents` = list of agents |
|
||||
| `PROXY_AGENT_UUID` | _(none)_ | Single agent UUID (required if `mode=agent`) |
|
||||
| `PROXY_AGENT_UUIDS` | _(none)_ | Comma-separated agent UUIDs (required if `mode=agents`) |
|
||||
| `PROXY_AGENT_DELAY_MS` | `5000` | Delay (ms) between each agent collection to avoid rate-limiting |
|
||||
|
||||
### Scheduling
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `PROXY_CRON_SCHEDULE` | `*/5 * * * *` | Cron expression for collection interval |
|
||||
| `PROXY_CAPACITY_LOG_INTERVAL_MS` | `86400000` | How often to log DB capacity usage (default: 24h) |
|
||||
|
||||
### Limits
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `PROXY_FLOW_LIMIT` | `1000000` | Max flows to fetch per agent per cycle |
|
||||
| `PROXY_PORT` | `4000` | REST API listen port |
|
||||
|
||||
### Netify API
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `NETIFY_INFORMATICS_BASE_URL` | `https://informatics.netify.ai/api/v1` | Netify API base URL |
|
||||
|
||||
---
|
||||
|
||||
## Docker Run Example
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--name backone_proxy \
|
||||
-p 4000:4000 \
|
||||
-e MONGODB_URI=mongodb://host.docker.internal:27017/backone_dpi \
|
||||
-e NETIFY_SITE_UUIDS=site-uuid-1,site-uuid-2 \
|
||||
-e NETIFY_TOKEN=your-jwt-token \
|
||||
-e PROXY_COLLECT_MODE=agents \
|
||||
-e PROXY_AGENT_UUIDS=agent-uuid-1,agent-uuid-2,agent-uuid-3 \
|
||||
backone-proxy
|
||||
```
|
||||
|
||||
## Docker Compose Example
|
||||
|
||||
```yaml
|
||||
services:
|
||||
proxy:
|
||||
build: ./proxy
|
||||
container_name: backone_proxy
|
||||
restart: always
|
||||
ports:
|
||||
- "4000:4000"
|
||||
environment:
|
||||
- MONGODB_URI=mongodb://mongodb:27017/backone_dpi
|
||||
- PROXY_PORT=4000
|
||||
- PROXY_COLLECT_MODE=all
|
||||
- PROXY_COLLECT_MODE=${PROXY_COLLECT_MODE:-all}
|
||||
- PROXY_AGENT_UUID=${PROXY_AGENT_UUID:-}
|
||||
- PROXY_AGENT_UUIDS=${PROXY_AGENT_UUIDS:-}
|
||||
- PROXY_AGENT_DELAY_MS=${PROXY_AGENT_DELAY_MS:-5000}
|
||||
- PROXY_CRON_SCHEDULE=${PROXY_CRON_SCHEDULE:-*/5 * * * *}
|
||||
- NETIFY_SITE_UUIDS=${NETIFY_SITE_UUIDS}
|
||||
- NETIFY_TOKEN=${NETIFY_TOKEN}
|
||||
- NETIFY_INFORMATICS_BASE_URL=${NETIFY_INFORMATICS_BASE_URL:-https://informatics.netify.ai/api/v1}
|
||||
```
|
||||
|
||||
## REST API Endpoints
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|---|---|---|
|
||||
| `GET` | `/health` | Liveness check (MongoDB status) |
|
||||
| `GET` | `/status` | Scheduler status, mode, last run |
|
||||
| `GET` | `/agents` | List agent UUIDs in MongoDB |
|
||||
| `POST` | `/collect/all` | Manual trigger — all agents |
|
||||
| `POST` | `/collect/:uuid` | Manual trigger — single agent |
|
||||
| `POST` | `/collect/agents` | Manual trigger — multiple agents `{"uuids":[...], "delay_ms":5000}` |
|
||||
| `GET` | `/latest` | Latest data from all collections (debug) |
|
||||
| `GET` | `/domain-details?domain=...` | IP/MAC details for a domain |
|
||||
|
||||
## Health Check
|
||||
|
||||
```bash
|
||||
curl http://localhost:4000/health
|
||||
```
|
||||
# BackOne DPI Proxy — Deployment Reference
|
||||
|
||||
Standalone Docker image for collecting Netify DPI data and writing to MongoDB.
|
||||
|
||||
---
|
||||
|
||||
## Environment Variables
|
||||
|
||||
### Required
|
||||
|
||||
| Variable | Description |
|
||||
|---|---|
|
||||
| `NETIFY_SITE_UUIDS` | Comma-separated Netify site UUIDs (or single `NETIFY_SITE_UUID`) |
|
||||
| `NETIFY_TOKEN` | Netify JWT token (or `NETIFY_JWT_TOKEN` / `NETIFY_API_KEY`) |
|
||||
|
||||
### MongoDB
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `MONGODB_URI` | `mongodb://127.0.0.1:27017/backone_dpi` | MongoDB connection string |
|
||||
|
||||
### Collection Mode
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `PROXY_COLLECT_MODE` | `all` | `all` = all agents, `agent` = single agent, `agents` = list of agents |
|
||||
| `PROXY_AGENT_UUID` | _(none)_ | Single agent UUID (required if `mode=agent`) |
|
||||
| `PROXY_AGENT_UUIDS` | _(none)_ | Comma-separated agent UUIDs (required if `mode=agents`) |
|
||||
| `PROXY_AGENT_DELAY_MS` | `5000` | Delay (ms) between each agent collection to avoid rate-limiting |
|
||||
|
||||
### Scheduling
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `PROXY_CRON_SCHEDULE` | `*/5 * * * *` | Cron expression for collection interval |
|
||||
| `PROXY_CAPACITY_LOG_INTERVAL_MS` | `86400000` | How often to log DB capacity usage (default: 24h) |
|
||||
|
||||
### Limits
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `PROXY_FLOW_LIMIT` | `1000000` | Max flows to fetch per agent per cycle |
|
||||
| `PROXY_PORT` | `4000` | REST API listen port |
|
||||
|
||||
### Netify API
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `NETIFY_INFORMATICS_BASE_URL` | `https://informatics.netify.ai/api/v1` | Netify API base URL |
|
||||
|
||||
---
|
||||
|
||||
## Docker Run Example
|
||||
|
||||
```bash
|
||||
docker run -d \
|
||||
--name backone_proxy \
|
||||
-p 4000:4000 \
|
||||
-e MONGODB_URI=mongodb://host.docker.internal:27017/backone_dpi \
|
||||
-e NETIFY_SITE_UUIDS=site-uuid-1,site-uuid-2 \
|
||||
-e NETIFY_TOKEN=your-jwt-token \
|
||||
-e PROXY_COLLECT_MODE=agents \
|
||||
-e PROXY_AGENT_UUIDS=agent-uuid-1,agent-uuid-2,agent-uuid-3 \
|
||||
backone-proxy
|
||||
```
|
||||
|
||||
## Docker Compose Example
|
||||
|
||||
```yaml
|
||||
services:
|
||||
proxy:
|
||||
build: ./proxy
|
||||
container_name: backone_proxy
|
||||
restart: always
|
||||
ports:
|
||||
- "4000:4000"
|
||||
environment:
|
||||
- MONGODB_URI=mongodb://mongodb:27017/backone_dpi
|
||||
- PROXY_PORT=4000
|
||||
- PROXY_COLLECT_MODE=all
|
||||
- PROXY_COLLECT_MODE=${PROXY_COLLECT_MODE:-all}
|
||||
- PROXY_AGENT_UUID=${PROXY_AGENT_UUID:-}
|
||||
- PROXY_AGENT_UUIDS=${PROXY_AGENT_UUIDS:-}
|
||||
- PROXY_AGENT_DELAY_MS=${PROXY_AGENT_DELAY_MS:-5000}
|
||||
- PROXY_CRON_SCHEDULE=${PROXY_CRON_SCHEDULE:-*/5 * * * *}
|
||||
- NETIFY_SITE_UUIDS=${NETIFY_SITE_UUIDS}
|
||||
- NETIFY_TOKEN=${NETIFY_TOKEN}
|
||||
- NETIFY_INFORMATICS_BASE_URL=${NETIFY_INFORMATICS_BASE_URL:-https://informatics.netify.ai/api/v1}
|
||||
```
|
||||
|
||||
## REST API Endpoints
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|---|---|---|
|
||||
| `GET` | `/health` | Liveness check (MongoDB status) |
|
||||
| `GET` | `/status` | Scheduler status, mode, last run |
|
||||
| `GET` | `/agents` | List agent UUIDs in MongoDB |
|
||||
| `POST` | `/collect/all` | Manual trigger — all agents |
|
||||
| `POST` | `/collect/:uuid` | Manual trigger — single agent |
|
||||
| `POST` | `/collect/agents` | Manual trigger — multiple agents `{"uuids":[...], "delay_ms":5000}` |
|
||||
| `GET` | `/latest` | Latest data from all collections (debug) |
|
||||
| `GET` | `/domain-details?domain=...` | IP/MAC details for a domain |
|
||||
|
||||
## Health Check
|
||||
|
||||
```bash
|
||||
curl http://localhost:4000/health
|
||||
```
|
||||
@@ -0,0 +1,34 @@
|
||||
// check_device.js - Detailed check of 10.6.10.44 records
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
|
||||
const mongoose = require('mongoose');
|
||||
|
||||
async function run() {
|
||||
await mongoose.connect(process.env.MONGODB_URI || 'mongodb://localhost:27017/backone');
|
||||
const db = mongoose.connection.db;
|
||||
|
||||
// Get all records for 10.6.10.44
|
||||
const docs = await db.collection('devicestats')
|
||||
.find({ ip_address: '10.6.10.44' })
|
||||
.sort({ timestamp: 1 })
|
||||
.toArray();
|
||||
|
||||
console.log(`Total docs for 10.6.10.44: ${docs.length}`);
|
||||
docs.forEach((d, i) => {
|
||||
console.log(`\n--- Doc ${i + 1} ---`);
|
||||
console.log(' _id: ', d._id);
|
||||
console.log(' agent_uuid: ', d.agent_uuid);
|
||||
console.log(' timestamp: ', d.timestamp);
|
||||
console.log(' created_at: ', d.created_at);
|
||||
console.log(' updated_at: ', d.updated_at);
|
||||
console.log(' download: ', d.download);
|
||||
console.log(' device_label:', d.device_label);
|
||||
});
|
||||
|
||||
// Check if there are different agent_uuids
|
||||
const agents = [...new Set(docs.map(d => d.agent_uuid))];
|
||||
console.log('\nDistinct agent_uuids for this IP:', agents);
|
||||
|
||||
await mongoose.disconnect();
|
||||
}
|
||||
run().catch(err => { console.error(err.message); process.exit(1); });
|
||||
@@ -0,0 +1,63 @@
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
|
||||
const mongoose = require('mongoose');
|
||||
|
||||
const MONGODB_URI = process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017/backone_dpi';
|
||||
const SIAB = '6681452d_9cae_4ff4_8ae8_0d504774265e';
|
||||
|
||||
mongoose.connect(MONGODB_URI).then(async () => {
|
||||
const db = mongoose.connection.db;
|
||||
const since24h = new Date(Date.now() - 24 * 3600000);
|
||||
const since7d = new Date(Date.now() - 7 * 24 * 3600000);
|
||||
|
||||
// Count site-level summary docs
|
||||
const count24h = await db.collection('summaries').countDocuments({
|
||||
site_uuid: SIAB, agent_uuid: null, timestamp: { $gte: since24h }
|
||||
});
|
||||
const countAll = await db.collection('summaries').countDocuments({
|
||||
site_uuid: SIAB, agent_uuid: null
|
||||
});
|
||||
|
||||
// Sum bandwidth for last 24h (site-level, agent_uuid: null)
|
||||
const agg24h = await db.collection('summaries').aggregate([
|
||||
{ $match: { site_uuid: SIAB, agent_uuid: null, timestamp: { $gte: since24h } } },
|
||||
{ $group: { _id: null, totalDown: { $sum: '$bandwidth_down' }, totalUp: { $sum: '$bandwidth_up' }, count: { $sum: 1 } } }
|
||||
]).toArray();
|
||||
|
||||
// Sum bandwidth ALL time (site-level)
|
||||
const aggAll = await db.collection('summaries').aggregate([
|
||||
{ $match: { site_uuid: SIAB, agent_uuid: null } },
|
||||
{ $group: { _id: null, totalDown: { $sum: '$bandwidth_down' }, totalUp: { $sum: '$bandwidth_up' }, count: { $sum: 1 } } }
|
||||
]).toArray();
|
||||
|
||||
// Oldest and newest
|
||||
const oldest = await db.collection('summaries').findOne({ site_uuid: SIAB, agent_uuid: null }, { sort: { timestamp: 1 }, projection: { timestamp: 1 } });
|
||||
const newest = await db.collection('summaries').findOne({ site_uuid: SIAB, agent_uuid: null }, { sort: { timestamp: -1 }, projection: { timestamp: 1, bandwidth_down: 1, bandwidth_up: 1 } });
|
||||
|
||||
console.log('\n=== MongoDB Summary Check (SIAB site) ===');
|
||||
console.log('Total site-level docs:', countAll);
|
||||
console.log('Site-level docs in last 24h:', count24h);
|
||||
console.log('\nBandwidth SUM (last 24h):');
|
||||
console.log(' Down:', agg24h[0] ? (agg24h[0].totalDown / 1024 / 1024).toFixed(2) + ' MB' : '0 MB');
|
||||
console.log(' Up :', agg24h[0] ? (agg24h[0].totalUp / 1024 / 1024).toFixed(2) + ' MB' : '0 MB');
|
||||
console.log('\nBandwidth SUM (ALL time):');
|
||||
console.log(' Down:', aggAll[0] ? (aggAll[0].totalDown / 1024 / 1024).toFixed(2) + ' MB' : '0 MB');
|
||||
console.log(' Up :', aggAll[0] ? (aggAll[0].totalUp / 1024 / 1024).toFixed(2) + ' MB' : '0 MB');
|
||||
console.log('\nOldest entry :', oldest?.timestamp);
|
||||
console.log('Newest entry :', newest?.timestamp);
|
||||
console.log('Latest bandwidth_down per 5min:', newest ? (newest.bandwidth_down / 1024 / 1024).toFixed(4) + ' MB' : 'N/A');
|
||||
console.log('Latest bandwidth_up per 5min :', newest ? (newest.bandwidth_up / 1024 / 1024).toFixed(4) + ' MB' : 'N/A');
|
||||
|
||||
// Also check flow data for comparison
|
||||
const flowAgg = await db.collection('flows').aggregate([
|
||||
{ $match: { site_uuid: SIAB, timestamp: { $gte: since24h } } },
|
||||
{ $group: { _id: null, totalDown: { $sum: '$download' }, totalUp: { $sum: '$upload' }, count: { $sum: 1 } } }
|
||||
]).toArray();
|
||||
console.log('\nFlow-level bandwidth (last 24h from flows collection):');
|
||||
console.log(' Down:', flowAgg[0] ? (flowAgg[0].totalDown / 1024 / 1024).toFixed(2) + ' MB' : '0 MB');
|
||||
console.log(' Up :', flowAgg[0] ? (flowAgg[0].totalUp / 1024 / 1024).toFixed(2) + ' MB' : '0 MB');
|
||||
console.log(' Flow count:', flowAgg[0]?.count || 0);
|
||||
console.log('=====================================\n');
|
||||
|
||||
process.exit(0);
|
||||
}).catch(e => { console.error(e.message); process.exit(1); });
|
||||
@@ -0,0 +1,73 @@
|
||||
// proxy/clean_devicestat_duplicates.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// One-time cleanup script to deduplicate historical DeviceStat records.
|
||||
// Keeps only the LATEST document per (agent_uuid, ip_address) pair,
|
||||
// removing all older duplicates accumulated before the upsert fix.
|
||||
//
|
||||
// Usage: node proxy/clean_devicestat_duplicates.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
|
||||
|
||||
const mongoose = require('mongoose');
|
||||
const MONGODB_URI = process.env.MONGODB_URI || 'mongodb://localhost:27017/backone';
|
||||
|
||||
const DeviceStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date },
|
||||
agent_uuid: { type: String },
|
||||
site_uuid: { type: String },
|
||||
ip_address: { type: String },
|
||||
mac_address: { type: String },
|
||||
device_label: String,
|
||||
device_type: String,
|
||||
os_label: String,
|
||||
manufacturer: String,
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
last_seen: String,
|
||||
}, { timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' } });
|
||||
|
||||
const DeviceStat = mongoose.model('DeviceStat', DeviceStatSchema);
|
||||
|
||||
async function run() {
|
||||
console.log('[Cleanup] Connecting to MongoDB...');
|
||||
await mongoose.connect(MONGODB_URI);
|
||||
console.log('[Cleanup] Connected.');
|
||||
|
||||
// Find all unique (agent_uuid, ip_address) combinations
|
||||
const groups = await DeviceStat.aggregate([
|
||||
{ $group: {
|
||||
_id: { agent_uuid: '$agent_uuid', ip_address: '$ip_address' },
|
||||
ids: { $push: '$_id' },
|
||||
timestamps: { $push: '$timestamp' },
|
||||
count: { $sum: 1 },
|
||||
}},
|
||||
{ $match: { count: { $gt: 1 } } },
|
||||
]);
|
||||
|
||||
console.log(`[Cleanup] Found ${groups.length} (agent_uuid, ip_address) pairs with duplicates.`);
|
||||
let totalDeleted = 0;
|
||||
|
||||
for (const group of groups) {
|
||||
// Sort the ids by matching timestamps - keep the latest
|
||||
const paired = group.ids.map((id, i) => ({ id, ts: group.timestamps[i] }));
|
||||
paired.sort((a, b) => new Date(b.ts) - new Date(a.ts));
|
||||
|
||||
// Keep the first (newest), delete the rest
|
||||
const toDelete = paired.slice(1).map(p => p.id);
|
||||
const result = await DeviceStat.deleteMany({ _id: { $in: toDelete } });
|
||||
totalDeleted += result.deletedCount;
|
||||
}
|
||||
|
||||
const remaining = await DeviceStat.countDocuments();
|
||||
console.log(`[Cleanup] Done. Deleted ${totalDeleted} duplicate DeviceStat records.`);
|
||||
console.log(`[Cleanup] Remaining DeviceStat documents: ${remaining}`);
|
||||
await mongoose.disconnect();
|
||||
}
|
||||
|
||||
run().catch(err => {
|
||||
console.error('[Cleanup] Fatal error:', err.message);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -8,8 +8,8 @@ const mongoose = require('mongoose');
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.join(__dirname, '../../../..', '.env.local') });
|
||||
|
||||
const SIAB_UUID = '6681452d_9cae_4ff4_8ae8_0d504774265e';
|
||||
const NEXUS_UUID = 'd7902405_0dc2_458b_8584_ed4d24b64f24';
|
||||
const SIAB_UUID = '6681452d_9cae_4ff4_8ae8_0d504774265e';
|
||||
const OFFICE_UUID = '1959bb55_045b_47c7_bbdd_f33b7db197b9';
|
||||
|
||||
// Definitive SIAB agent list (from most recent collector run)
|
||||
const SIAB_AGENTS = ['F6-2V-DT-8A', 'YW-6I-61-LL', '2F-TF-1D-GK', '1R-79-J9-YE', '8A-V3-PB-85'];
|
||||
@@ -27,13 +27,13 @@ async function cleanup() {
|
||||
for (const colName of collections) {
|
||||
const col = db.collection(colName);
|
||||
|
||||
// 1. Delete SIAB agents that are stored under NEXUS site_uuid
|
||||
// 1. Delete SIAB agents that are stored under OFFICE site_uuid
|
||||
const r1 = await col.deleteMany({
|
||||
site_uuid: NEXUS_UUID,
|
||||
site_uuid: OFFICE_UUID,
|
||||
agent_uuid: { $in: SIAB_AGENTS }
|
||||
});
|
||||
if (r1.deletedCount > 0) {
|
||||
console.log(`[${colName}] Removed ${r1.deletedCount} docs (SIAB agents from NEXUS)`);
|
||||
console.log(`[${colName}] Removed ${r1.deletedCount} docs (SIAB agents from OFFICE)`);
|
||||
totalDeleted += r1.deletedCount;
|
||||
}
|
||||
|
||||
|
||||
+263
-262
@@ -1,262 +1,263 @@
|
||||
// proxy/collector.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Core data collection logic for the BackOne Proxy Server
|
||||
// Supports 2 modes: ALL Agents and ONE Agent by UUID
|
||||
// All data is stored in MongoDB, tagged with agent_uuid + site_uuid.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
|
||||
|
||||
const netify = require('./netifyClient');
|
||||
const { collectSecondaryTelemetry } = require('./collectorHelper');
|
||||
const {
|
||||
collectDevicesAndApps,
|
||||
collectFlows,
|
||||
collectThreats,
|
||||
collectEvents
|
||||
} = require('./collectorHelperDpi2');
|
||||
|
||||
const { Summary, AppStat } = require('./models/Schemas');
|
||||
const { pruneOldData } = require('./dataRetention');
|
||||
|
||||
const SITE_UUIDS_STR = process.env.NETIFY_SITE_UUIDS || process.env.NETIFY_SITE_UUID;
|
||||
const SITE_UUIDS = SITE_UUIDS_STR ? SITE_UUIDS_STR.split(',').map(s => s.trim()).filter(Boolean) : [];
|
||||
|
||||
async function collectForAgent(agentUuid, timestamp, siteUuid) {
|
||||
const label = agentUuid || 'GLOBAL';
|
||||
console.log(`[Collector] → Fetching data for Agent: ${label}`);
|
||||
|
||||
try {
|
||||
// 1. Summary / Bandwidth (including derived speed, packet_drops, and peak_flow_rate)
|
||||
const summary = await netify.fetchBandwidthSummary(1440, agentUuid, siteUuid);
|
||||
if (summary) {
|
||||
let download_speed = 0;
|
||||
let upload_speed = 0;
|
||||
let packet_drops = 0;
|
||||
let peak_flow_rate = summary.active_flows || 0;
|
||||
|
||||
try {
|
||||
const prev = await Summary.findOne({ agent_uuid: agentUuid }).sort({ timestamp: -1 }).lean();
|
||||
if (prev && prev.timestamp) {
|
||||
const timeDiffSec = (timestamp.getTime() - new Date(prev.timestamp).getTime()) / 1000;
|
||||
if (timeDiffSec > 0) {
|
||||
const bytesDiffDown = Math.max(0, summary.bandwidth_down - (prev.bandwidth_down || 0));
|
||||
const bytesDiffUp = Math.max(0, summary.bandwidth_up - (prev.bandwidth_up || 0));
|
||||
download_speed = bytesDiffDown / timeDiffSec;
|
||||
upload_speed = bytesDiffUp / timeDiffSec;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[Collector] Error calculating summary speeds:', err.message);
|
||||
}
|
||||
|
||||
packet_drops = Math.floor((summary.active_flows || 0) * 0.015);
|
||||
peak_flow_rate = Math.floor((summary.active_flows || 0) * 1.18);
|
||||
|
||||
const activeFlows = summary.active_flows || 0;
|
||||
const totalBandwidth = (summary.bandwidth_down || 0) + (summary.bandwidth_up || 0);
|
||||
|
||||
const cpu_usage = Math.min(98, Math.max(1.2, parseFloat((2.5 + (activeFlows * 0.04) + (totalBandwidth / 10000000)).toFixed(2))));
|
||||
const memory_usage = Math.min(99, Math.max(10.5, parseFloat((15.4 + (activeFlows * 0.02) + (totalBandwidth / 25000000)).toFixed(2))));
|
||||
const queue_depth = Math.max(0, Math.floor((activeFlows * 0.15) + (totalBandwidth / 5000000)));
|
||||
|
||||
await new Summary({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: siteUuid,
|
||||
...summary,
|
||||
download_speed,
|
||||
upload_speed,
|
||||
packet_drops,
|
||||
peak_flow_rate,
|
||||
cpu_usage,
|
||||
memory_usage,
|
||||
queue_depth
|
||||
}).save();
|
||||
console.log(`[Collector] ✓ Summary saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2. Top Apps
|
||||
const apps = await netify.fetchTopApps(1440, 200, agentUuid, siteUuid);
|
||||
if (apps && apps.length > 0) {
|
||||
const appDocs = apps.map(app => ({
|
||||
timestamp, agent_uuid: agentUuid, site_uuid: siteUuid,
|
||||
app_label: app.application?.label || 'Unknown',
|
||||
download: app.download || 0, upload: app.upload || 0, flows: app.flows || 0,
|
||||
}));
|
||||
await AppStat.insertMany(appDocs);
|
||||
console.log(`[Collector] ✓ ${appDocs.length} apps saved for ${label}`);
|
||||
}
|
||||
|
||||
// Collect Secondary Telemetry (categories, TLS, countries, DHCP, User Agents, BitTorrent)
|
||||
await collectSecondaryTelemetry(agentUuid, timestamp, siteUuid, netify, label);
|
||||
|
||||
// 2. Devices & App Records
|
||||
const ipToMacMap = await collectDevicesAndApps(agentUuid, timestamp, siteUuid, netify, label);
|
||||
|
||||
// 3. Flows
|
||||
await collectFlows(agentUuid, timestamp, siteUuid, netify, label, ipToMacMap);
|
||||
|
||||
// 5. Threats
|
||||
await collectThreats(agentUuid, timestamp, siteUuid, netify, label);
|
||||
|
||||
// 6. Events
|
||||
await collectEvents(agentUuid, timestamp, siteUuid, netify, label);
|
||||
|
||||
return { success: true, agent_uuid: agentUuid };
|
||||
} catch (err) {
|
||||
console.error(`[Collector] ✗ Error collecting for ${label}:`, err.message);
|
||||
return { success: false, agent_uuid: agentUuid, error: err.message };
|
||||
}
|
||||
}
|
||||
|
||||
async function collectAllAgents() {
|
||||
const timestamp = new Date();
|
||||
const timeString = timestamp.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
|
||||
console.log(`[Collector] === MODE: ALL AGENTS === Started at ${timeString}`);
|
||||
|
||||
const results = [];
|
||||
let totalAgents = 0;
|
||||
|
||||
if (SITE_UUIDS.length === 0) {
|
||||
console.warn('[Collector] No NETIFY_SITE_UUIDS configured.');
|
||||
return { success: false, mode: 'all', message: 'No sites configured', results: [] };
|
||||
}
|
||||
|
||||
// Track agent UUIDs already assigned to a site to prevent cross-site duplication.
|
||||
// The Netify /data/stats/top/agent/download endpoint is org-level and can return
|
||||
// the same agent for multiple site queries. Each agent must belong to exactly one site.
|
||||
const processedAgentUuids = new Set();
|
||||
|
||||
for (const siteUuid of SITE_UUIDS) {
|
||||
console.log(`[Collector] Fetching agents for Site: ${siteUuid}`);
|
||||
const rawAgents = await netify.fetchAgents(siteUuid);
|
||||
if (!rawAgents || rawAgents.length === 0) {
|
||||
console.warn(`[Collector] No agents found for site ${siteUuid}.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Deduplicate: only keep agents not yet seen in a previous site this cycle
|
||||
const agents = rawAgents.filter(a => {
|
||||
if (processedAgentUuids.has(a.uuid)) {
|
||||
console.log(`[Collector] Skipping agent ${a.uuid} — already assigned to another site.`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
if (agents.length === 0) {
|
||||
console.warn(`[Collector] No unique agents for site ${siteUuid} (all were already assigned). Skipping.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Register these agents as belonging to this site
|
||||
for (const agent of agents) processedAgentUuids.add(agent.uuid);
|
||||
|
||||
totalAgents += agents.length;
|
||||
console.log(`[Collector] Processing ${agents.length} agents for site ${siteUuid}: ${agents.map(a => a.uuid).join(', ')}`);
|
||||
|
||||
// ── Site-Level Summary (Pilihan A) ─────────────────────────────────────
|
||||
// Collect bandwidth at site level (no agentUuid filter) so numbers match
|
||||
// Netify portal exactly and avoid double-counting across agents.
|
||||
try {
|
||||
console.log(`[Collector] → Fetching site-level summary for site: ${siteUuid}`);
|
||||
const siteSummary = await netify.fetchBandwidthSummary(1440, null, siteUuid);
|
||||
if (siteSummary) {
|
||||
let download_speed = 0;
|
||||
let upload_speed = 0;
|
||||
|
||||
try {
|
||||
const prev = await Summary.findOne({ agent_uuid: null, site_uuid: siteUuid }).sort({ timestamp: -1 }).lean();
|
||||
if (prev && prev.timestamp) {
|
||||
const timeDiffSec = (timestamp.getTime() - new Date(prev.timestamp).getTime()) / 1000;
|
||||
if (timeDiffSec > 0) {
|
||||
const bytesDiffDown = Math.max(0, siteSummary.bandwidth_down - (prev.bandwidth_down || 0));
|
||||
const bytesDiffUp = Math.max(0, siteSummary.bandwidth_up - (prev.bandwidth_up || 0));
|
||||
download_speed = bytesDiffDown / timeDiffSec;
|
||||
upload_speed = bytesDiffUp / timeDiffSec;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[Collector] Error calculating site summary speeds:', err.message);
|
||||
}
|
||||
|
||||
const activeFlows = siteSummary.active_flows || 0;
|
||||
const totalBandwidth = (siteSummary.bandwidth_down || 0) + (siteSummary.bandwidth_up || 0);
|
||||
const packet_drops = Math.floor(activeFlows * 0.015);
|
||||
const peak_flow_rate = Math.floor(activeFlows * 1.18);
|
||||
const cpu_usage = Math.min(98, Math.max(1.2, parseFloat((2.5 + (activeFlows * 0.04) + (totalBandwidth / 10000000)).toFixed(2))));
|
||||
const memory_usage = Math.min(99, Math.max(10.5, parseFloat((15.4 + (activeFlows * 0.02) + (totalBandwidth / 25000000)).toFixed(2))));
|
||||
const queue_depth = Math.max(0, Math.floor((activeFlows * 0.15) + (totalBandwidth / 5000000)));
|
||||
|
||||
await new Summary({
|
||||
timestamp,
|
||||
agent_uuid: null, // null = site-level aggregate (bukan per-agent)
|
||||
site_uuid: siteUuid,
|
||||
...siteSummary,
|
||||
download_speed,
|
||||
upload_speed,
|
||||
packet_drops,
|
||||
peak_flow_rate,
|
||||
cpu_usage,
|
||||
memory_usage,
|
||||
queue_depth
|
||||
}).save();
|
||||
console.log(`[Collector] ✓ Site-level summary saved for site: ${siteUuid} | Down: ${(siteSummary.bandwidth_down / 1e9).toFixed(2)} GB | Up: ${(siteSummary.bandwidth_up / 1e9).toFixed(2)} GB | Flows: ${siteSummary.active_flows?.toLocaleString()}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[Collector] ✗ Failed to save site-level summary for ${siteUuid}:`, err.message);
|
||||
}
|
||||
|
||||
for (const agent of agents) {
|
||||
const result = await collectForAgent(agent.uuid, timestamp, siteUuid);
|
||||
results.push({ ...result, agent_label: agent.label, site_uuid: siteUuid });
|
||||
}
|
||||
}
|
||||
|
||||
const successful = results.filter(r => r.success).length;
|
||||
console.log(`[Collector] === ALL AGENTS DONE === ${successful}/${totalAgents} successful across ${SITE_UUIDS.length} sites`);
|
||||
|
||||
await pruneOldData().catch(err => console.error('[Collector] [Retention] error:', err.message));
|
||||
|
||||
return { success: true, mode: 'all', agents_count: totalAgents, successful };
|
||||
}
|
||||
|
||||
async function collectSpecificAgent(agentUuid, siteUuid = SITE_UUIDS[0]) {
|
||||
const timestamp = new Date();
|
||||
const timeString = timestamp.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
|
||||
console.log(`[Collector] === MODE: SPECIFIC AGENT ${agentUuid} === Started at ${timeString}`);
|
||||
const result = await collectForAgent(agentUuid, timestamp, siteUuid);
|
||||
|
||||
await pruneOldData().catch(err => console.error('[Collector] [Retention] error:', err.message));
|
||||
|
||||
return { success: result.success, mode: 'specific', agent_uuid: agentUuid };
|
||||
}
|
||||
|
||||
async function collectSpecificAgents(agentUuids, siteUuid = SITE_UUIDS[0], delayMs = 5000) {
|
||||
const timestamp = new Date();
|
||||
const timeString = timestamp.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
|
||||
console.log(`[Collector] === MODE: SPECIFIC AGENTS [${agentUuids.join(', ')}] === Started at ${timeString}`);
|
||||
const results = [];
|
||||
for (let i = 0; i < agentUuids.length; i++) {
|
||||
if (i > 0) {
|
||||
console.log(`[Collector] Waiting ${delayMs}ms before next agent...`);
|
||||
await new Promise(resolve => setTimeout(resolve, delayMs));
|
||||
}
|
||||
const result = await collectForAgent(agentUuids[i], timestamp, siteUuid);
|
||||
results.push(result);
|
||||
}
|
||||
const successful = results.filter(r => r.success).length;
|
||||
console.log(`[Collector] === SPECIFIC AGENTS DONE === ${successful}/${agentUuids.length} successful`);
|
||||
|
||||
await pruneOldData().catch(err => console.error('[Collector] [Retention] error:', err.message));
|
||||
|
||||
return { success: true, mode: 'specific_agents', agents_count: agentUuids.length, successful, results };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
collectAllAgents,
|
||||
collectSpecificAgent,
|
||||
collectSpecificAgents
|
||||
};
|
||||
// proxy/collector.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Core data collection logic for the BackOne Proxy Server
|
||||
// Supports 2 modes: ALL Agents and ONE Agent by UUID
|
||||
// All data is stored in MongoDB, tagged with agent_uuid + site_uuid.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
|
||||
|
||||
const netify = require('./netifyClient');
|
||||
const { collectSecondaryTelemetry } = require('./collectorHelper');
|
||||
const { collectDevicesAndApps, collectFlows } = require('./collectorHelperDpi2');
|
||||
const { collectThreats, collectEvents } = require('./collectorHelperDpi3');
|
||||
|
||||
const { Summary, AppStat, AgentRegistry } = require('./models/Schemas');
|
||||
const { pruneOldData } = require('./dataRetention');
|
||||
|
||||
const SITE_UUIDS_STR = process.env.NETIFY_SITE_UUIDS || process.env.NETIFY_SITE_UUID;
|
||||
const SITE_UUIDS = SITE_UUIDS_STR ? SITE_UUIDS_STR.split(',').map(s => s.trim()).filter(Boolean) : [];
|
||||
|
||||
async function collectForAgent(agentUuid, timestamp, siteUuid) {
|
||||
const label = agentUuid || 'GLOBAL';
|
||||
console.log(`[Collector] → Fetching data for Agent: ${label}`);
|
||||
|
||||
try {
|
||||
// 1. Summary / Bandwidth (including derived speed, packet_drops, and peak_flow_rate)
|
||||
const summary = await netify.fetchBandwidthSummary(5, agentUuid, siteUuid);
|
||||
if (summary) {
|
||||
let download_speed = 0;
|
||||
let upload_speed = 0;
|
||||
let packet_drops = 0;
|
||||
let peak_flow_rate = summary.active_flows || 0;
|
||||
|
||||
try {
|
||||
const prev = await Summary.findOne({ agent_uuid: agentUuid }).sort({ timestamp: -1 }).lean();
|
||||
const timeDiffSec = prev && prev.timestamp ? (timestamp.getTime() - new Date(prev.timestamp).getTime()) / 1000 : 300;
|
||||
const activeTimeDiff = timeDiffSec > 0 ? timeDiffSec : 300;
|
||||
download_speed = summary.bandwidth_down / activeTimeDiff;
|
||||
upload_speed = summary.bandwidth_up / activeTimeDiff;
|
||||
} catch (err) {
|
||||
console.error('[Collector] Error calculating summary speeds:', err.message);
|
||||
}
|
||||
|
||||
packet_drops = Math.floor((summary.active_flows || 0) * 0.015);
|
||||
peak_flow_rate = Math.floor((summary.active_flows || 0) * 1.18);
|
||||
|
||||
const activeFlows = summary.active_flows || 0;
|
||||
const totalBandwidth = (summary.bandwidth_down || 0) + (summary.bandwidth_up || 0);
|
||||
|
||||
const cpu_usage = Math.min(98, Math.max(1.2, parseFloat((2.5 + (activeFlows * 0.04) + (totalBandwidth / 10000000)).toFixed(2))));
|
||||
const memory_usage = Math.min(99, Math.max(10.5, parseFloat((15.4 + (activeFlows * 0.02) + (totalBandwidth / 25000000)).toFixed(2))));
|
||||
const queue_depth = Math.max(0, Math.floor((activeFlows * 0.15) + (totalBandwidth / 5000000)));
|
||||
|
||||
await new Summary({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: siteUuid,
|
||||
...summary,
|
||||
download_speed,
|
||||
upload_speed,
|
||||
packet_drops,
|
||||
peak_flow_rate,
|
||||
cpu_usage,
|
||||
memory_usage,
|
||||
queue_depth
|
||||
}).save();
|
||||
console.log(`[Collector] ✓ Summary saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2. Top Apps
|
||||
const apps = await netify.fetchTopApps(5, 200, agentUuid, siteUuid);
|
||||
if (apps && apps.length > 0) {
|
||||
const appDocs = apps.map(app => ({
|
||||
timestamp, agent_uuid: agentUuid, site_uuid: siteUuid,
|
||||
app_label: app.application?.label || 'Unknown',
|
||||
download: app.download || 0, upload: app.upload || 0, flows: app.flows || 0,
|
||||
}));
|
||||
await AppStat.insertMany(appDocs);
|
||||
console.log(`[Collector] ✓ ${appDocs.length} apps saved for ${label}`);
|
||||
}
|
||||
|
||||
// Collect Secondary Telemetry (categories, TLS, countries, DHCP, User Agents, BitTorrent)
|
||||
await collectSecondaryTelemetry(agentUuid, timestamp, siteUuid, netify, label);
|
||||
|
||||
// 2. Devices & App Records
|
||||
const ipToMacMap = await collectDevicesAndApps(agentUuid, timestamp, siteUuid, netify, label);
|
||||
|
||||
// 3. Flows
|
||||
await collectFlows(agentUuid, timestamp, siteUuid, netify, label, ipToMacMap);
|
||||
|
||||
// 5. Threats
|
||||
await collectThreats(agentUuid, timestamp, siteUuid, netify, label);
|
||||
|
||||
// 6. Events
|
||||
await collectEvents(agentUuid, timestamp, siteUuid, netify, label);
|
||||
|
||||
return { success: true, agent_uuid: agentUuid };
|
||||
} catch (err) {
|
||||
console.error(`[Collector] ✗ Error collecting for ${label}:`, err.message);
|
||||
return { success: false, agent_uuid: agentUuid, error: err.message };
|
||||
}
|
||||
}
|
||||
|
||||
async function collectAllAgents() {
|
||||
const timestamp = new Date();
|
||||
const timeString = timestamp.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
|
||||
console.log(`[Collector] === MODE: ALL AGENTS === Started at ${timeString}`);
|
||||
|
||||
const results = [];
|
||||
let totalAgents = 0;
|
||||
|
||||
if (SITE_UUIDS.length === 0) {
|
||||
console.warn('[Collector] No NETIFY_SITE_UUIDS configured.');
|
||||
return { success: false, mode: 'all', message: 'No sites configured', results: [] };
|
||||
}
|
||||
|
||||
// Track agent UUIDs already assigned to a site to prevent cross-site duplication.
|
||||
// The Netify /data/stats/top/agent/download endpoint is org-level and can return
|
||||
// the same agent for multiple site queries. Each agent must belong to exactly one site.
|
||||
const processedAgentUuids = new Set();
|
||||
|
||||
for (const siteUuid of SITE_UUIDS) {
|
||||
console.log(`[Collector] Fetching agents for Site: ${siteUuid}`);
|
||||
const rawAgents = await netify.fetchAgents(siteUuid);
|
||||
if (!rawAgents || rawAgents.length === 0) {
|
||||
console.warn(`[Collector] No agents found for site ${siteUuid}.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Deduplicate: only keep agents not yet seen in a previous site this cycle
|
||||
const agents = rawAgents.filter(a => {
|
||||
if (processedAgentUuids.has(a.uuid)) {
|
||||
console.log(`[Collector] Skipping agent ${a.uuid} — already assigned to another site.`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
if (agents.length === 0) {
|
||||
console.warn(`[Collector] No unique agents for site ${siteUuid} (all were already assigned). Skipping.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Register these agents as belonging to this site
|
||||
for (const agent of agents) processedAgentUuids.add(agent.uuid);
|
||||
|
||||
// ── Upsert all agents into agent_registry collection ───────────────────
|
||||
// This ensures ALL agents appear in the frontend even with no telemetry data.
|
||||
await Promise.allSettled(agents.map(a =>
|
||||
AgentRegistry.findOneAndUpdate(
|
||||
{ uuid: a.uuid },
|
||||
{
|
||||
$set: {
|
||||
uuid: a.uuid,
|
||||
serial: a.serial || a.uuid,
|
||||
label: a.label,
|
||||
site_uuid: siteUuid,
|
||||
provisioned: a.provisioned ?? true,
|
||||
activated: a.activated ?? false,
|
||||
last_seen_at: a.last_seen_at ?? null,
|
||||
netify_id: a.id ?? null,
|
||||
}
|
||||
},
|
||||
{ upsert: true, new: true }
|
||||
)
|
||||
));
|
||||
console.log(`[Collector] ✓ ${agents.length} agents upserted into registry for site ${siteUuid}`);
|
||||
|
||||
totalAgents += agents.length;
|
||||
console.log(`[Collector] Processing ${agents.length} agents for site ${siteUuid}: ${agents.map(a => a.uuid).join(', ')}`);
|
||||
|
||||
// ── Site-Level Summary (Pilihan A) ─────────────────────────────────────
|
||||
// Collect bandwidth at site level (no agentUuid filter) so numbers match
|
||||
// Netify portal exactly and avoid double-counting across agents.
|
||||
try {
|
||||
const siteSummary = await netify.fetchBandwidthSummary(5, null, siteUuid);
|
||||
if (siteSummary) {
|
||||
let download_speed = 0;
|
||||
let upload_speed = 0;
|
||||
|
||||
try {
|
||||
const prev = await Summary.findOne({ agent_uuid: null, site_uuid: siteUuid }).sort({ timestamp: -1 }).lean();
|
||||
const timeDiffSec = prev && prev.timestamp ? (timestamp.getTime() - new Date(prev.timestamp).getTime()) / 1000 : 300;
|
||||
const activeTimeDiff = timeDiffSec > 0 ? timeDiffSec : 300;
|
||||
download_speed = siteSummary.bandwidth_down / activeTimeDiff;
|
||||
upload_speed = siteSummary.bandwidth_up / activeTimeDiff;
|
||||
} catch (err) {
|
||||
console.error('[Collector] Error calculating site summary speeds:', err.message);
|
||||
}
|
||||
|
||||
const activeFlows = siteSummary.active_flows || 0;
|
||||
const totalBandwidth = (siteSummary.bandwidth_down || 0) + (siteSummary.bandwidth_up || 0);
|
||||
const packet_drops = Math.floor(activeFlows * 0.015);
|
||||
const peak_flow_rate = Math.floor(activeFlows * 1.18);
|
||||
const cpu_usage = Math.min(98, Math.max(1.2, parseFloat((2.5 + (activeFlows * 0.04) + (totalBandwidth / 10000000)).toFixed(2))));
|
||||
const memory_usage = Math.min(99, Math.max(10.5, parseFloat((15.4 + (activeFlows * 0.02) + (totalBandwidth / 25000000)).toFixed(2))));
|
||||
const queue_depth = Math.max(0, Math.floor((activeFlows * 0.15) + (totalBandwidth / 5000000)));
|
||||
|
||||
await new Summary({
|
||||
timestamp,
|
||||
agent_uuid: null, // null = site-level aggregate (bukan per-agent)
|
||||
site_uuid: siteUuid,
|
||||
...siteSummary,
|
||||
download_speed,
|
||||
upload_speed,
|
||||
packet_drops,
|
||||
peak_flow_rate,
|
||||
cpu_usage,
|
||||
memory_usage,
|
||||
queue_depth
|
||||
}).save();
|
||||
console.log(`[Collector] ✓ Site-level summary saved for site: ${siteUuid} | Down: ${(siteSummary.bandwidth_down / 1e9).toFixed(2)} GB | Up: ${(siteSummary.bandwidth_up / 1e9).toFixed(2)} GB | Flows: ${siteSummary.active_flows?.toLocaleString()}`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`[Collector] ✗ Failed to save site-level summary for ${siteUuid}:`, err.message);
|
||||
}
|
||||
|
||||
for (const agent of agents) {
|
||||
const result = await collectForAgent(agent.uuid, timestamp, siteUuid);
|
||||
results.push({ ...result, agent_label: agent.label, site_uuid: siteUuid });
|
||||
}
|
||||
}
|
||||
|
||||
const successful = results.filter(r => r.success).length;
|
||||
console.log(`[Collector] === ALL AGENTS DONE === ${successful}/${totalAgents} successful across ${SITE_UUIDS.length} sites`);
|
||||
|
||||
await pruneOldData().catch(err => console.error('[Collector] [Retention] error:', err.message));
|
||||
|
||||
return { success: true, mode: 'all', agents_count: totalAgents, successful };
|
||||
}
|
||||
|
||||
async function collectSpecificAgent(agentUuid, siteUuid = SITE_UUIDS[0]) {
|
||||
const result = await collectSpecificAgents([agentUuid], siteUuid, 0);
|
||||
return { success: result.successful > 0, mode: 'specific', agent_uuid: agentUuid };
|
||||
}
|
||||
|
||||
async function collectSpecificAgents(agentUuids, siteUuid = SITE_UUIDS[0], delayMs = 5000) {
|
||||
const timestamp = new Date();
|
||||
const timeString = timestamp.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
|
||||
console.log(`[Collector] === MODE: SPECIFIC AGENTS [${agentUuids.join(', ')}] === Started at ${timeString}`);
|
||||
const results = [];
|
||||
for (let i = 0; i < agentUuids.length; i++) {
|
||||
if (i > 0) {
|
||||
console.log(`[Collector] Waiting ${delayMs}ms before next agent...`);
|
||||
await new Promise(resolve => setTimeout(resolve, delayMs));
|
||||
}
|
||||
const result = await collectForAgent(agentUuids[i], timestamp, siteUuid);
|
||||
results.push(result);
|
||||
}
|
||||
const successful = results.filter(r => r.success).length;
|
||||
console.log(`[Collector] === SPECIFIC AGENTS DONE === ${successful}/${agentUuids.length} successful`);
|
||||
|
||||
await pruneOldData().catch(err => console.error('[Collector] [Retention] error:', err.message));
|
||||
|
||||
return { success: true, mode: 'specific_agents', agents_count: agentUuids.length, successful, results };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
collectAllAgents,
|
||||
collectSpecificAgent,
|
||||
collectSpecificAgents
|
||||
};
|
||||
+167
-167
@@ -1,167 +1,167 @@
|
||||
// proxy/collectorHelper.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Supplementary Telemetry collection steps for BackOne Proxy Server.
|
||||
// Split from collector.js to satisfy the 256-line file size limit.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const {
|
||||
AppCategoryStat,
|
||||
TlsVersionStat,
|
||||
TlsCipherStat,
|
||||
TlsSecurityStat,
|
||||
CountryStat,
|
||||
ProtocolStat,
|
||||
SslSubjectAltNameStat,
|
||||
SniHostnameStat,
|
||||
SslServerCnStat,
|
||||
QuicHostnameStat,
|
||||
} = require('./models/Schemas');
|
||||
|
||||
async function collectSecondaryTelemetry(agentUuid, timestamp, SITE_UUID, netify, label) {
|
||||
try {
|
||||
// 2b. App Categories
|
||||
const categories = await netify.fetchTopAppCategories(1440, 50, agentUuid, SITE_UUID);
|
||||
if (categories && categories.length > 0) {
|
||||
const catDocs = categories.map(c => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
category_label: c.category_label,
|
||||
download: c.download || 0,
|
||||
upload: c.upload || 0,
|
||||
flows: c.flows || 0,
|
||||
}));
|
||||
await AppCategoryStat.insertMany(catDocs);
|
||||
console.log(`[Collector] ✓ ${catDocs.length} categories saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2c. TLS Versions
|
||||
const tlsVersions = await netify.fetchTlsVersions(1440, 50, agentUuid, SITE_UUID);
|
||||
if (tlsVersions && tlsVersions.length > 0) {
|
||||
const tvDocs = tlsVersions.map(v => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
tls_version: v.tls_version,
|
||||
download: v.download || 0,
|
||||
upload: v.upload || 0,
|
||||
flows: v.flows || 0,
|
||||
}));
|
||||
await TlsVersionStat.insertMany(tvDocs);
|
||||
console.log(`[Collector] ✓ ${tvDocs.length} TLS versions saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2d. TLS Ciphers
|
||||
const tlsCiphers = await netify.fetchTlsCiphers(1440, 50, agentUuid, SITE_UUID);
|
||||
if (tlsCiphers && tlsCiphers.length > 0) {
|
||||
const tcDocs = tlsCiphers.map(c => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
tls_cipher: c.tls_cipher,
|
||||
download: c.download || 0,
|
||||
upload: c.upload || 0,
|
||||
flows: c.flows || 0,
|
||||
}));
|
||||
await TlsCipherStat.insertMany(tcDocs);
|
||||
console.log(`[Collector] ✓ ${tcDocs.length} TLS ciphers saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2e. TLS Security
|
||||
const tlsSecurity = await netify.fetchTlsSecurity(1440, 50, agentUuid, SITE_UUID);
|
||||
if (tlsSecurity && tlsSecurity.length > 0) {
|
||||
const tsDocs = tlsSecurity.map(s => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
tls_security: s.tls_security,
|
||||
download: s.download || 0,
|
||||
upload: s.upload || 0,
|
||||
flows: s.flows || 0,
|
||||
}));
|
||||
await TlsSecurityStat.insertMany(tsDocs);
|
||||
console.log(`[Collector] ✓ ${tsDocs.length} TLS security stats saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2f. Top Countries
|
||||
const countries = await netify.fetchTopCountries(1440, 100, agentUuid, SITE_UUID);
|
||||
if (countries && countries.length > 0) {
|
||||
const coDocs = countries.map(c => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
country_code: c.country_code,
|
||||
country_name: c.country_name || '',
|
||||
download: c.download || 0,
|
||||
upload: c.upload || 0,
|
||||
flows: c.flows || 0,
|
||||
}));
|
||||
await CountryStat.insertMany(coDocs);
|
||||
console.log(`[Collector] ✓ ${coDocs.length} countries saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2g. Top Protocols
|
||||
const protocols = await netify.fetchTopProtocols(1440, 50, agentUuid, SITE_UUID);
|
||||
if (protocols && protocols.length > 0) {
|
||||
const protoDocs = protocols.map(p => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
protocol_label: p.protocol_label,
|
||||
download: p.download || 0,
|
||||
upload: p.upload || 0,
|
||||
flows: p.flows || 0,
|
||||
}));
|
||||
await ProtocolStat.insertMany(protoDocs);
|
||||
console.log(`[Collector] ✓ ${protoDocs.length} protocols saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2h. SNI Hostnames
|
||||
const snis = await netify.fetchSniHostnames(1440, 10000, agentUuid, SITE_UUID);
|
||||
if (snis && snis.length > 0) {
|
||||
const sniDocs = snis.map(s => ({
|
||||
timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
|
||||
sni_hostname: (s.sni_hostname && String(s.sni_hostname).trim() !== '') ? s.sni_hostname : 'Unknown',
|
||||
download: s.download || 0, upload: s.upload || 0, flows: s.flows || 0,
|
||||
}));
|
||||
await SniHostnameStat.insertMany(sniDocs);
|
||||
console.log(`[Collector] ✓ ${sniDocs.length} SNI hostnames saved for ${label}`);
|
||||
}
|
||||
|
||||
/* -- COMMENTED OUT DUE TO NETIFY API HTTP 422 (UNSUPPORTED TIER) --
|
||||
// 2i. SSL Server Common Names
|
||||
const cns = await netify.fetchSslServerCn(1440, 50, agentUuid);
|
||||
if (cns && cns.length > 0) {
|
||||
const cnDocs = cns.map(c => ({
|
||||
timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
|
||||
ssl_server_cn: c.ssl_server_cn, download: c.download || 0, upload: c.upload || 0, flows: c.flows || 0,
|
||||
}));
|
||||
await SslServerCnStat.insertMany(cnDocs);
|
||||
console.log(`[Collector] ✓ ${cnDocs.length} SSL Server CNs saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2j. QUIC Hostnames
|
||||
const quics = await netify.fetchQuicHostnames(1440, 50, agentUuid);
|
||||
if (quics && quics.length > 0) {
|
||||
const quicDocs = quics.map(q => ({
|
||||
timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
|
||||
quic_hostname: q.quic_hostname, download: q.download || 0, upload: q.upload || 0, flows: q.flows || 0,
|
||||
}));
|
||||
await QuicHostnameStat.insertMany(quicDocs);
|
||||
console.log(`[Collector] ✓ ${quicDocs.length} QUIC hostnames saved for ${label}`);
|
||||
}
|
||||
*/
|
||||
|
||||
// NOTE: The following API fields are not supported on this subscription (HTTP 422):
|
||||
// dhcp_class, http_useragent, bittorrent_info_hash, ssl_subject_alt_name
|
||||
// These sections are intentionally skipped to avoid wasted API calls.
|
||||
// Re-enable when API access is upgraded to a tier that supports these fields.
|
||||
|
||||
} catch (err) {
|
||||
console.error(`[CollectorHelper] Error saving secondary telemetry:`, err.message);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
collectSecondaryTelemetry,
|
||||
};
|
||||
// proxy/collectorHelper.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Supplementary Telemetry collection steps for BackOne Proxy Server.
|
||||
// Split from collector.js to satisfy the 256-line file size limit.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const {
|
||||
AppCategoryStat,
|
||||
TlsVersionStat,
|
||||
TlsCipherStat,
|
||||
TlsSecurityStat,
|
||||
CountryStat,
|
||||
ProtocolStat,
|
||||
SslSubjectAltNameStat,
|
||||
SniHostnameStat,
|
||||
SslServerCnStat,
|
||||
QuicHostnameStat,
|
||||
} = require('./models/Schemas');
|
||||
|
||||
async function collectSecondaryTelemetry(agentUuid, timestamp, SITE_UUID, netify, label) {
|
||||
try {
|
||||
// 2b. App Categories
|
||||
const categories = await netify.fetchTopAppCategories(5, 50, agentUuid, SITE_UUID);
|
||||
if (categories && categories.length > 0) {
|
||||
const catDocs = categories.map(c => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
category_label: c.category_label,
|
||||
download: c.download || 0,
|
||||
upload: c.upload || 0,
|
||||
flows: c.flows || 0,
|
||||
}));
|
||||
await AppCategoryStat.insertMany(catDocs);
|
||||
console.log(`[Collector] ✓ ${catDocs.length} categories saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2c. TLS Versions
|
||||
const tlsVersions = await netify.fetchTlsVersions(5, 50, agentUuid, SITE_UUID);
|
||||
if (tlsVersions && tlsVersions.length > 0) {
|
||||
const tvDocs = tlsVersions.map(v => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
tls_version: v.tls_version,
|
||||
download: v.download || 0,
|
||||
upload: v.upload || 0,
|
||||
flows: v.flows || 0,
|
||||
}));
|
||||
await TlsVersionStat.insertMany(tvDocs);
|
||||
console.log(`[Collector] ✓ ${tvDocs.length} TLS versions saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2d. TLS Ciphers
|
||||
const tlsCiphers = await netify.fetchTlsCiphers(5, 50, agentUuid, SITE_UUID);
|
||||
if (tlsCiphers && tlsCiphers.length > 0) {
|
||||
const tcDocs = tlsCiphers.map(c => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
tls_cipher: c.tls_cipher,
|
||||
download: c.download || 0,
|
||||
upload: c.upload || 0,
|
||||
flows: c.flows || 0,
|
||||
}));
|
||||
await TlsCipherStat.insertMany(tcDocs);
|
||||
console.log(`[Collector] ✓ ${tcDocs.length} TLS ciphers saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2e. TLS Security
|
||||
const tlsSecurity = await netify.fetchTlsSecurity(5, 50, agentUuid, SITE_UUID);
|
||||
if (tlsSecurity && tlsSecurity.length > 0) {
|
||||
const tsDocs = tlsSecurity.map(s => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
tls_security: s.tls_security,
|
||||
download: s.download || 0,
|
||||
upload: s.upload || 0,
|
||||
flows: s.flows || 0,
|
||||
}));
|
||||
await TlsSecurityStat.insertMany(tsDocs);
|
||||
console.log(`[Collector] ✓ ${tsDocs.length} TLS security stats saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2f. Top Countries
|
||||
const countries = await netify.fetchTopCountries(5, 100, agentUuid, SITE_UUID);
|
||||
if (countries && countries.length > 0) {
|
||||
const coDocs = countries.map(c => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
country_code: c.country_code,
|
||||
country_name: c.country_name || '',
|
||||
download: c.download || 0,
|
||||
upload: c.upload || 0,
|
||||
flows: c.flows || 0,
|
||||
}));
|
||||
await CountryStat.insertMany(coDocs);
|
||||
console.log(`[Collector] ✓ ${coDocs.length} countries saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2g. Top Protocols
|
||||
const protocols = await netify.fetchTopProtocols(5, 50, agentUuid, SITE_UUID);
|
||||
if (protocols && protocols.length > 0) {
|
||||
const protoDocs = protocols.map(p => ({
|
||||
timestamp,
|
||||
agent_uuid: agentUuid,
|
||||
site_uuid: SITE_UUID,
|
||||
protocol_label: p.protocol_label,
|
||||
download: p.download || 0,
|
||||
upload: p.upload || 0,
|
||||
flows: p.flows || 0,
|
||||
}));
|
||||
await ProtocolStat.insertMany(protoDocs);
|
||||
console.log(`[Collector] ✓ ${protoDocs.length} protocols saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2h. SNI Hostnames
|
||||
const snis = await netify.fetchSniHostnames(5, 10000, agentUuid, SITE_UUID);
|
||||
if (snis && snis.length > 0) {
|
||||
const sniDocs = snis.map(s => ({
|
||||
timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
|
||||
sni_hostname: (s.sni_hostname && String(s.sni_hostname).trim() !== '') ? s.sni_hostname : 'Unknown',
|
||||
download: s.download || 0, upload: s.upload || 0, flows: s.flows || 0,
|
||||
}));
|
||||
await SniHostnameStat.insertMany(sniDocs);
|
||||
console.log(`[Collector] ✓ ${sniDocs.length} SNI hostnames saved for ${label}`);
|
||||
}
|
||||
|
||||
/* -- COMMENTED OUT DUE TO NETIFY API HTTP 422 (UNSUPPORTED TIER) --
|
||||
// 2i. SSL Server Common Names
|
||||
const cns = await netify.fetchSslServerCn(1440, 50, agentUuid);
|
||||
if (cns && cns.length > 0) {
|
||||
const cnDocs = cns.map(c => ({
|
||||
timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
|
||||
ssl_server_cn: c.ssl_server_cn, download: c.download || 0, upload: c.upload || 0, flows: c.flows || 0,
|
||||
}));
|
||||
await SslServerCnStat.insertMany(cnDocs);
|
||||
console.log(`[Collector] ✓ ${cnDocs.length} SSL Server CNs saved for ${label}`);
|
||||
}
|
||||
|
||||
// 2j. QUIC Hostnames
|
||||
const quics = await netify.fetchQuicHostnames(1440, 50, agentUuid);
|
||||
if (quics && quics.length > 0) {
|
||||
const quicDocs = quics.map(q => ({
|
||||
timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
|
||||
quic_hostname: q.quic_hostname, download: q.download || 0, upload: q.upload || 0, flows: q.flows || 0,
|
||||
}));
|
||||
await QuicHostnameStat.insertMany(quicDocs);
|
||||
console.log(`[Collector] ✓ ${quicDocs.length} QUIC hostnames saved for ${label}`);
|
||||
}
|
||||
*/
|
||||
|
||||
// NOTE: The following API fields are not supported on this subscription (HTTP 422):
|
||||
// dhcp_class, http_useragent, bittorrent_info_hash, ssl_subject_alt_name
|
||||
// These sections are intentionally skipped to avoid wasted API calls.
|
||||
// Re-enable when API access is upgraded to a tier that supports these fields.
|
||||
|
||||
} catch (err) {
|
||||
console.error(`[CollectorHelper] Error saving secondary telemetry:`, err.message);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
collectSecondaryTelemetry,
|
||||
};
|
||||
@@ -14,7 +14,7 @@ const {
|
||||
} = require('./deviceResolver');
|
||||
|
||||
async function collectDevicesAndApps(agentUuid, timestamp, SITE_UUID, netify, label) {
|
||||
const devices = await netify.fetchDiscoveredDevices(1440, 500, agentUuid, SITE_UUID);
|
||||
const devices = await netify.fetchDiscoveredDevices(5, 500, agentUuid, SITE_UUID);
|
||||
const ipToMacMap = {};
|
||||
|
||||
if (devices && devices.length > 0) {
|
||||
@@ -56,7 +56,7 @@ async function collectDevicesAndApps(agentUuid, timestamp, SITE_UUID, netify, la
|
||||
let deviceAppCount = 0;
|
||||
for (let i = 0; i < topDevices.length; i += 5) {
|
||||
const batch = topDevices.slice(i, i + 5);
|
||||
const results = await Promise.allSettled(batch.map(d => netify.fetchDeviceApps(d.ip_address, 1440, 50, agentUuid, SITE_UUID)));
|
||||
const results = await Promise.allSettled(batch.map(d => netify.fetchDeviceApps(d.ip_address, 5, 50, agentUuid, SITE_UUID)));
|
||||
const appDocs = [];
|
||||
results.forEach((res, idx) => {
|
||||
if (res.status === 'fulfilled' && Array.isArray(res.value)) {
|
||||
@@ -81,7 +81,7 @@ async function collectDevicesAndApps(agentUuid, timestamp, SITE_UUID, netify, la
|
||||
}
|
||||
|
||||
async function collectFlows(agentUuid, timestamp, SITE_UUID, netify, label, ipToMacMap) {
|
||||
// Netify API has a hard limit of 1,000,000 for settings_limit.
|
||||
// Netify API has a hard limit of 1,000,000 for settings_limit. Use 1000000 as default per rule.
|
||||
const flowLimit = parseInt(process.env.PROXY_FLOW_LIMIT || '1000000');
|
||||
const flows = await netify.fetchFlows(flowLimit, agentUuid, SITE_UUID);
|
||||
if (flows && flows.length > 0) {
|
||||
@@ -115,16 +115,34 @@ async function collectFlows(agentUuid, timestamp, SITE_UUID, netify, label, ipTo
|
||||
const blacklistedCategories = new Set(blacklistRules.filter(r => r.type === 'category').map(r => r.value.toLowerCase()));
|
||||
const blacklistedDomains = new Set(blacklistRules.filter(r => r.type === 'domain').map(r => r.value.toLowerCase()));
|
||||
|
||||
const flowIdsInBatch = flowDocs.map(f => f.flow_id).filter(Boolean);
|
||||
const existingFlowThreats = new Set(
|
||||
await Threat.find({ flow_id: { $in: flowIdsInBatch } }).distinct('flow_id')
|
||||
);
|
||||
|
||||
const threatDocs = [];
|
||||
const eventDocs = [];
|
||||
|
||||
for (const f of flowDocs) {
|
||||
let isViolation = false;
|
||||
let categoryLabel = "";
|
||||
|
||||
// Check if domain is blacklisted
|
||||
if (f.domain && blacklistedDomains.has(f.domain.toLowerCase())) {
|
||||
isViolation = true;
|
||||
} else if (f.app_label && blacklistedDomains.has(f.app_label.toLowerCase())) {
|
||||
isViolation = true;
|
||||
for (const r of blacklistRules) {
|
||||
if (r.type === 'domain') {
|
||||
const val = r.value.toLowerCase();
|
||||
// Direct domain match
|
||||
if (f.domain && f.domain.toLowerCase().includes(val)) {
|
||||
isViolation = true;
|
||||
break;
|
||||
}
|
||||
// Main domain part match against app label (e.g. "google" from "google.com")
|
||||
const mainDomainPart = val.split('.')[0];
|
||||
if (mainDomainPart && f.app_label && f.app_label.toLowerCase().includes(mainDomainPart)) {
|
||||
isViolation = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Look up app details to check category
|
||||
@@ -138,7 +156,7 @@ async function collectFlows(agentUuid, timestamp, SITE_UUID, netify, label, ipTo
|
||||
}
|
||||
}
|
||||
|
||||
if (isViolation) {
|
||||
if (isViolation && !existingFlowThreats.has(f.flow_id)) {
|
||||
threatDocs.push({
|
||||
timestamp,
|
||||
agent_uuid: f.agent_uuid,
|
||||
@@ -150,7 +168,21 @@ async function collectFlows(agentUuid, timestamp, SITE_UUID, netify, label, ipTo
|
||||
dst_port: f.dst_port,
|
||||
protocol: f.protocol,
|
||||
description: `Access to blacklisted app/domain: ${f.app_label} (${f.domain || 'N/A'})${categoryLabel ? ' - Category: ' + categoryLabel : ''}`,
|
||||
event_at: new Date().toISOString()
|
||||
event_at: new Date().toISOString(),
|
||||
flow_id: f.flow_id
|
||||
});
|
||||
|
||||
eventDocs.push({
|
||||
timestamp,
|
||||
agent_uuid: f.agent_uuid,
|
||||
site_uuid: f.site_uuid,
|
||||
event_type: "blacklist_violation",
|
||||
severity: "Warning",
|
||||
description: `Access to blacklisted app/domain: ${f.app_label} (${f.domain || 'N/A'})${categoryLabel ? ' - Category: ' + categoryLabel : ''}`,
|
||||
ip_address: f.src_ip,
|
||||
mac_address: f.src_mac,
|
||||
event_at: new Date(),
|
||||
flow_id: f.flow_id
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -159,6 +191,10 @@ async function collectFlows(agentUuid, timestamp, SITE_UUID, netify, label, ipTo
|
||||
await Threat.insertMany(threatDocs);
|
||||
console.log(`[Collector] ✓ ${threatDocs.length} blacklist policy violation threats recorded for ${label}`);
|
||||
}
|
||||
if (eventDocs.length > 0) {
|
||||
await Event.insertMany(eventDocs);
|
||||
console.log(`[Collector] ✓ ${eventDocs.length} blacklist policy violation events recorded for ${label}`);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[Collector] Blacklist detection failed:', err.message);
|
||||
@@ -169,58 +205,7 @@ async function collectFlows(agentUuid, timestamp, SITE_UUID, netify, label, ipTo
|
||||
}
|
||||
}
|
||||
|
||||
async function collectThreats(agentUuid, timestamp, SITE_UUID, netify, label) {
|
||||
const threats = await netify.fetchCyberThreats(agentUuid, SITE_UUID);
|
||||
if (threats && threats.length > 0) {
|
||||
const threatDocs = threats.map(t => ({
|
||||
timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
|
||||
threat_type: t.threat_type || 'Unknown Threat', severity: t.severity || 'Medium',
|
||||
src_ip: t.src_ip, dst_ip: t.dst_ip, dst_port: t.dst_port, protocol: t.protocol,
|
||||
description: t.description, event_at: t.event_at || new Date().toISOString(),
|
||||
}));
|
||||
await Threat.insertMany(threatDocs);
|
||||
console.log(`[Collector] ✓ ${threatDocs.length} threats saved for ${label}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function collectEvents(agentUuid, timestamp, SITE_UUID, netify, label) {
|
||||
const events = await netify.fetchEvents(100, agentUuid, SITE_UUID);
|
||||
if (events && events.length > 0) {
|
||||
const eventIds = events.map(e => e.event_id).filter(id => id !== null);
|
||||
const existing = await Event.find({ site_uuid: SITE_UUID, event_id: { $in: eventIds } }).distinct('event_id');
|
||||
const existingSet = new Set(existing);
|
||||
|
||||
const macToAgentMap = {};
|
||||
const eventMacs = [...new Set(events.map(e => e.mac_address).filter(Boolean))];
|
||||
if (eventMacs.length > 0) {
|
||||
const storedDevices = await DeviceStat.find(
|
||||
{ site_uuid: SITE_UUID, mac_address: { $in: eventMacs } },
|
||||
{ mac_address: 1, agent_uuid: 1 }
|
||||
).lean();
|
||||
for (const d of storedDevices) {
|
||||
if (d.mac_address && d.agent_uuid) macToAgentMap[d.mac_address] = d.agent_uuid;
|
||||
}
|
||||
}
|
||||
|
||||
const eventDocs = events.filter(e => e.event_id === null || !existingSet.has(e.event_id)).map(e => {
|
||||
const resolvedAgentUuid = (e.mac_address && macToAgentMap[e.mac_address]) || agentUuid;
|
||||
return {
|
||||
timestamp, agent_uuid: resolvedAgentUuid, site_uuid: SITE_UUID,
|
||||
event_id: e.event_id, event_type: e.event_type, severity: e.severity,
|
||||
description: e.description, category_label: e.category_label,
|
||||
ip_address: e.ip_address, mac_address: e.mac_address, event_at: e.event_at,
|
||||
};
|
||||
});
|
||||
if (eventDocs.length > 0) {
|
||||
await Event.insertMany(eventDocs);
|
||||
console.log(`[Collector] ✓ ${eventDocs.length} new events saved for ${label}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
collectDevicesAndApps,
|
||||
collectFlows,
|
||||
collectThreats,
|
||||
collectEvents
|
||||
collectFlows
|
||||
};
|
||||
@@ -0,0 +1,61 @@
|
||||
// proxy/collectorHelperDpi3.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Supplementary Telemetry collection steps for threats and events.
|
||||
// Split from collectorHelperDpi2.js to satisfy the 256-line file size limit.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const { DeviceStat, Threat, Event } = require('./models/Schemas');
|
||||
|
||||
async function collectThreats(agentUuid, timestamp, SITE_UUID, netify, label) {
|
||||
const threats = await netify.fetchCyberThreats(agentUuid, SITE_UUID);
|
||||
if (threats && threats.length > 0) {
|
||||
const threatDocs = threats.map(t => ({
|
||||
timestamp, agent_uuid: agentUuid, site_uuid: SITE_UUID,
|
||||
threat_type: t.threat_type || 'Unknown Threat', severity: t.severity || 'Medium',
|
||||
src_ip: t.src_ip, dst_ip: t.dst_ip, dst_port: t.dst_port, protocol: t.protocol,
|
||||
description: t.description, event_at: t.event_at || new Date().toISOString(),
|
||||
}));
|
||||
await Threat.insertMany(threatDocs);
|
||||
console.log(`[Collector] ✓ ${threatDocs.length} threats saved for ${label}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function collectEvents(agentUuid, timestamp, SITE_UUID, netify, label) {
|
||||
const events = await netify.fetchEvents(100, agentUuid, SITE_UUID);
|
||||
if (events && events.length > 0) {
|
||||
const eventIds = events.map(e => e.event_id).filter(id => id !== null);
|
||||
const existing = await Event.find({ site_uuid: SITE_UUID, event_id: { $in: eventIds } }).distinct('event_id');
|
||||
const existingSet = new Set(existing);
|
||||
|
||||
const macToAgentMap = {};
|
||||
const eventMacs = [...new Set(events.map(e => e.mac_address).filter(Boolean))];
|
||||
if (eventMacs.length > 0) {
|
||||
const storedDevices = await DeviceStat.find(
|
||||
{ site_uuid: SITE_UUID, mac_address: { $in: eventMacs } },
|
||||
{ mac_address: 1, agent_uuid: 1 }
|
||||
).lean();
|
||||
for (const d of storedDevices) {
|
||||
if (d.mac_address && d.agent_uuid) macToAgentMap[d.mac_address] = d.agent_uuid;
|
||||
}
|
||||
}
|
||||
|
||||
const eventDocs = events.filter(e => e.event_id === null || !existingSet.has(e.event_id)).map(e => {
|
||||
const resolvedAgentUuid = (e.mac_address && macToAgentMap[e.mac_address]) || agentUuid;
|
||||
return {
|
||||
timestamp, agent_uuid: resolvedAgentUuid, site_uuid: SITE_UUID,
|
||||
event_id: e.event_id, event_type: e.event_type, severity: e.severity,
|
||||
description: e.description, category_label: e.category_label,
|
||||
ip_address: e.ip_address, mac_address: e.mac_address, event_at: e.event_at,
|
||||
};
|
||||
});
|
||||
if (eventDocs.length > 0) {
|
||||
await Event.insertMany(eventDocs);
|
||||
console.log(`[Collector] ✓ ${eventDocs.length} new events saved for ${label}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
collectThreats,
|
||||
collectEvents
|
||||
};
|
||||
@@ -11,9 +11,9 @@ const Schemas = require('./models/Schemas');
|
||||
* Prune all time-series documents older than 7 days.
|
||||
*/
|
||||
async function pruneOldData() {
|
||||
const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
|
||||
const timeString = sevenDaysAgo.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
|
||||
console.log(`[Collector] [Retention] Checking for telemetry data older than 7 days (before ${timeString})...`);
|
||||
const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
|
||||
const timeString = thirtyDaysAgo.toLocaleString('id-ID', { timeZone: 'Asia/Jakarta' }) + ' WIB';
|
||||
console.log(`[Collector] [Retention] Checking for telemetry data older than 30 days (before ${timeString})...`);
|
||||
|
||||
// Prune from all collections in Schemas except CustomDeviceLabel
|
||||
const collections = Object.keys(Schemas).filter(name => name !== 'CustomDeviceLabel');
|
||||
@@ -22,7 +22,7 @@ async function pruneOldData() {
|
||||
try {
|
||||
const Model = Schemas[name];
|
||||
if (typeof Model.deleteMany === 'function') {
|
||||
const res = await Model.deleteMany({ timestamp: { $lt: sevenDaysAgo } });
|
||||
const res = await Model.deleteMany({ timestamp: { $lt: thirtyDaysAgo } });
|
||||
if (res.deletedCount > 0) {
|
||||
console.log(`[Collector] [Retention] ✓ Cleaned up ${res.deletedCount} old records from ${name}`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
// diagnostic.js - Run: node proxy/diagnostic.js
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
|
||||
const mongoose = require('mongoose');
|
||||
|
||||
const MONGODB_URI = process.env.MONGODB_URI || 'mongodb://localhost:27017/backone';
|
||||
|
||||
async function run() {
|
||||
await mongoose.connect(MONGODB_URI);
|
||||
const db = mongoose.connection.db;
|
||||
|
||||
// 1. Total count
|
||||
const total = await db.collection('devicestats').countDocuments();
|
||||
console.log('=== DeviceStat Total:', total);
|
||||
|
||||
// 2. Count for 10.6.10.44
|
||||
const specific = await db.collection('devicestats').countDocuments({ ip_address: '10.6.10.44' });
|
||||
console.log('=== Count for 10.6.10.44:', specific);
|
||||
|
||||
// 3. Sample doc for 10.6.10.44
|
||||
const sample = await db.collection('devicestats').findOne({ ip_address: '10.6.10.44' });
|
||||
console.log('=== Sample doc for 10.6.10.44:', JSON.stringify(sample, null, 2));
|
||||
|
||||
// 4. Duplicate groups (top 10)
|
||||
const dups = await db.collection('devicestats').aggregate([
|
||||
{ $group: { _id: { agent_uuid: '$agent_uuid', ip_address: '$ip_address' }, count: { $sum: 1 } } },
|
||||
{ $match: { count: { $gt: 1 } } },
|
||||
{ $sort: { count: -1 } },
|
||||
{ $limit: 10 }
|
||||
]).toArray();
|
||||
console.log('=== Top duplicate groups:', JSON.stringify(dups, null, 2));
|
||||
|
||||
// 5. Check what collection name is actually used
|
||||
const collections = await db.listCollections().toArray();
|
||||
console.log('=== Collections:', collections.map(c => c.name));
|
||||
|
||||
// 6. Check indexes on devicestats
|
||||
const indexes = await db.collection('devicestats').indexes();
|
||||
console.log('=== Indexes on devicestats:', JSON.stringify(indexes, null, 2));
|
||||
|
||||
await mongoose.disconnect();
|
||||
}
|
||||
|
||||
run().catch(err => { console.error('ERROR:', err.message); process.exit(1); });
|
||||
@@ -0,0 +1,79 @@
|
||||
// fix_devicestat_index.js
|
||||
// Creates a unique compound index on (agent_uuid, ip_address) in DeviceStat
|
||||
// and deduplicates any remaining duplicates before creating the index.
|
||||
// Run: node proxy/fix_devicestat_index.js
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
|
||||
const mongoose = require('mongoose');
|
||||
|
||||
const MONGODB_URI = process.env.MONGODB_URI || 'mongodb://localhost:27017/backone';
|
||||
|
||||
async function run() {
|
||||
console.log('[Fix] Connecting to MongoDB...');
|
||||
await mongoose.connect(MONGODB_URI);
|
||||
const db = mongoose.connection.db;
|
||||
const col = db.collection('devicestats');
|
||||
|
||||
// Step 1: Find all duplicates grouped by (agent_uuid, ip_address)
|
||||
console.log('[Fix] Scanning for duplicates...');
|
||||
const groups = await col.aggregate([
|
||||
{
|
||||
$group: {
|
||||
_id: { agent_uuid: '$agent_uuid', ip_address: '$ip_address' },
|
||||
ids: { $push: '$_id' },
|
||||
timestamps: { $push: '$timestamp' },
|
||||
count: { $sum: 1 },
|
||||
}
|
||||
},
|
||||
{ $match: { count: { $gt: 1 } } },
|
||||
]).toArray();
|
||||
|
||||
console.log(`[Fix] Found ${groups.length} duplicate groups.`);
|
||||
let deleted = 0;
|
||||
|
||||
for (const group of groups) {
|
||||
// Sort by timestamp descending — keep the newest
|
||||
const paired = group.ids.map((id, i) => ({ id, ts: new Date(group.timestamps[i] || 0) }));
|
||||
paired.sort((a, b) => b.ts - a.ts);
|
||||
const toDelete = paired.slice(1).map(p => p.id);
|
||||
const result = await col.deleteMany({ _id: { $in: toDelete } });
|
||||
deleted += result.deletedCount;
|
||||
}
|
||||
|
||||
console.log(`[Fix] Deleted ${deleted} duplicate documents.`);
|
||||
const remaining = await col.countDocuments();
|
||||
console.log(`[Fix] Remaining DeviceStat documents: ${remaining}`);
|
||||
|
||||
// Step 2: Drop old non-unique compound index if it exists
|
||||
try {
|
||||
await col.dropIndex('agent_uuid_1_timestamp_-1_ip_address_1');
|
||||
console.log('[Fix] Dropped old compound index.');
|
||||
} catch (e) {
|
||||
console.log('[Fix] Old index not found or already dropped:', e.message);
|
||||
}
|
||||
|
||||
// Step 3: Create UNIQUE compound index on (agent_uuid, ip_address)
|
||||
try {
|
||||
await col.createIndex(
|
||||
{ agent_uuid: 1, ip_address: 1 },
|
||||
{ unique: true, name: 'agent_uuid_1_ip_address_1_unique', background: true }
|
||||
);
|
||||
console.log('[Fix] Created unique index on (agent_uuid, ip_address).');
|
||||
} catch (e) {
|
||||
console.error('[Fix] Failed to create unique index:', e.message);
|
||||
}
|
||||
|
||||
// Step 4: Verify indexes
|
||||
const indexes = await col.indexes();
|
||||
console.log('[Fix] Current indexes:');
|
||||
indexes.forEach(idx => console.log(` - ${idx.name}: ${JSON.stringify(idx.key)} ${idx.unique ? '[UNIQUE]' : ''}`));
|
||||
|
||||
// Step 5: Verify 10.6.10.44
|
||||
const cnt = await col.countDocuments({ ip_address: '10.6.10.44' });
|
||||
console.log(`\n[Fix] Count for 10.6.10.44: ${cnt} (should be 1)`);
|
||||
|
||||
await mongoose.disconnect();
|
||||
console.log('[Fix] Done.');
|
||||
}
|
||||
|
||||
run().catch(err => { console.error('[Fix] Fatal:', err.message); process.exit(1); });
|
||||
+13
-5
@@ -1,10 +1,16 @@
|
||||
// proxy/index.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Polyfill global crypto for Node 18 compatibility (required by mongodb driver)
|
||||
if (typeof globalThis.crypto === 'undefined') {
|
||||
globalThis.crypto = require('crypto');
|
||||
}
|
||||
|
||||
// BackOne Proxy Server - Entry Point
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
|
||||
const envFile = process.env.NODE_ENV === 'production' ? '.env.production' : '.env.local';
|
||||
require('dotenv').config({ path: path.join(__dirname, '..', envFile) });
|
||||
|
||||
const express = require('express');
|
||||
const cors = require('cors');
|
||||
@@ -57,11 +63,13 @@ async function main() {
|
||||
console.log('╚════════════════════════════════════════════════╝\n');
|
||||
console.log(`[Proxy] Mode: ${process.env.PROXY_COLLECT_MODE || 'all'}`);
|
||||
|
||||
// Start REST API server first so liveness probes remain active
|
||||
app.listen(PORT, '0.0.0.0', () => {
|
||||
console.log(`\n🚀 Proxy REST API running at http://0.0.0.0:${PORT}`);
|
||||
// Bind to 127.0.0.1 in production — port 4000 must never be exposed externally
|
||||
const BIND_HOST = process.env.NODE_ENV === 'production' ? '127.0.0.1' : '0.0.0.0';
|
||||
app.listen(PORT, BIND_HOST, () => {
|
||||
console.log(`\n🚀 Proxy REST API running at http://${BIND_HOST}:${PORT}`);
|
||||
console.log(` GET /health → liveness check`);
|
||||
console.log(` GET /status → scheduler + DB status\n`);
|
||||
console.log(` GET /status → scheduler + DB status`);
|
||||
console.log(`🔒 Security : Bound to ${BIND_HOST} (internal only in production)\n`);
|
||||
});
|
||||
|
||||
const connected = await connectDB();
|
||||
|
||||
+199
-182
@@ -1,182 +1,199 @@
|
||||
// proxy/models/Schemas.js
|
||||
// MongoDB schemas shared between the proxy server (write) and backend (read).
|
||||
// Each document is tagged with agent_uuid + site_uuid for tenant isolation.
|
||||
//
|
||||
// IMPORTANT: Indexes are set for common query patterns:
|
||||
// - timestamp (for time-range queries)
|
||||
// - agent_uuid (for per-tenant filtering)
|
||||
// - site_uuid (for site-level aggregation)
|
||||
|
||||
const mongoose = require('mongoose');
|
||||
|
||||
const baseOptions = {
|
||||
timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
|
||||
};
|
||||
|
||||
// ─── Bandwidth Summary (per agent, per collection cycle) ───────────────────────
|
||||
const SummarySchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true }, // null = global/all agents
|
||||
site_uuid: { type: String, index: true },
|
||||
bandwidth_down: Number,
|
||||
bandwidth_up: Number,
|
||||
active_flows: Number,
|
||||
download_speed: Number,
|
||||
upload_speed: Number,
|
||||
total_devices: Number,
|
||||
total_threats: Number,
|
||||
packet_drops: Number,
|
||||
peak_flow_rate: Number,
|
||||
cpu_usage: Number,
|
||||
memory_usage: Number,
|
||||
queue_depth: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Top Applications (per agent) ─────────────────────────────────────────────
|
||||
const AppStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
app_label: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Protocol Statistics (per agent) ──────────────────────────────────────────
|
||||
const ProtocolStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
protocol_label: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Discovered Devices (per agent, includes IP + MAC + device info) ───────────
|
||||
const DeviceStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
ip_address: { type: String, required: true, index: true },
|
||||
mac_address: { type: String, index: true },
|
||||
device_label: String,
|
||||
device_type: String,
|
||||
os_label: String,
|
||||
manufacturer: String,
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
last_seen: String,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Network Flows (per agent) ─────────────────────────────────────────────────
|
||||
const FlowSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
flow_id: String,
|
||||
src_ip: { type: String, index: true },
|
||||
src_mac: { type: String, index: true }, // indexed for MAC-to-IP resolution in events
|
||||
dst_ip: { type: String, index: true },
|
||||
dst_port: Number,
|
||||
protocol: String,
|
||||
app_label: String,
|
||||
domain: String,
|
||||
download: Number,
|
||||
upload: Number,
|
||||
first_seen: String,
|
||||
last_seen: String,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Cyber Threats (per agent) ─────────────────────────────────────────────────
|
||||
const ThreatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
threat_type: String,
|
||||
severity: String,
|
||||
src_ip: String,
|
||||
dst_ip: String,
|
||||
dst_port: Number,
|
||||
protocol: String,
|
||||
description: String,
|
||||
event_at: String,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── App Categories (per agent) ───────────────────────────────────────────────
|
||||
const AppCategoryStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
category_label: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── System Events (per agent) ─────────────────────────────────────────────────
|
||||
const EventSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
event_id: Number,
|
||||
event_type: String,
|
||||
severity: String,
|
||||
description: String,
|
||||
category_label: String,
|
||||
ip_address: String,
|
||||
mac_address: String,
|
||||
event_at: Date,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Compound indexes for common dashboard queries ─────────────────────────────
|
||||
SummarySchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
AppStatSchema.index({ agent_uuid: 1, timestamp: -1, download: -1 });
|
||||
DeviceStatSchema.index({ agent_uuid: 1, ip_address: 1 }, { unique: true });
|
||||
FlowSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
FlowSchema.index({ agent_uuid: 1, flow_id: 1 });
|
||||
FlowSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
|
||||
ThreatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
AppCategoryStatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
EventSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
FlowSchema.index({ site_uuid: 1, src_mac: 1, timestamp: -1 }); // for MAC-to-IP resolution
|
||||
|
||||
// ── Per-Device Per-Application Stats ────────────────────────────────────────
|
||||
// Collected from DPI API: /data/stats/top/application/download with filter_local_ips
|
||||
// Allows showing "YouTube 134GB" in Device Detail modal per specific IP
|
||||
const DeviceAppStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
ip_address: { type: String, required: true, index: true },
|
||||
app_label: { type: String, required: true },
|
||||
app_id: Number,
|
||||
download: { type: Number, default: 0 },
|
||||
upload: { type: Number, default: 0 },
|
||||
flows: { type: Number, default: 0 },
|
||||
last_seen: String,
|
||||
}, baseOptions);
|
||||
DeviceAppStatSchema.index({ agent_uuid: 1, ip_address: 1, timestamp: -1 });
|
||||
DeviceAppStatSchema.index({ ip_address: 1, app_label: 1, timestamp: -1 });
|
||||
DeviceAppStatSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
|
||||
|
||||
const telemetrySchemas = require('./SchemasTelemetry');
|
||||
const auxSchemas = require('./SchemasAux');
|
||||
|
||||
module.exports = {
|
||||
Summary: mongoose.model('Summary', SummarySchema),
|
||||
AppStat: mongoose.model('AppStat', AppStatSchema),
|
||||
ProtocolStat:mongoose.model('ProtocolStat',ProtocolStatSchema),
|
||||
DeviceStat: mongoose.model('DeviceStat', DeviceStatSchema),
|
||||
DeviceAppStat: mongoose.model('DeviceAppStat', DeviceAppStatSchema),
|
||||
Flow: mongoose.model('Flow', FlowSchema),
|
||||
Threat: mongoose.model('Threat', ThreatSchema),
|
||||
AppCategoryStat: mongoose.model('AppCategoryStat', AppCategoryStatSchema),
|
||||
Event: mongoose.model('Event', EventSchema),
|
||||
...auxSchemas,
|
||||
...telemetrySchemas,
|
||||
};
|
||||
|
||||
|
||||
// proxy/models/Schemas.js
|
||||
// MongoDB schemas shared between the proxy server (write) and backend (read).
|
||||
// Each document is tagged with agent_uuid + site_uuid for tenant isolation.
|
||||
//
|
||||
// IMPORTANT: Indexes are set for common query patterns:
|
||||
// - timestamp (for time-range queries)
|
||||
// - agent_uuid (for per-tenant filtering)
|
||||
// - site_uuid (for site-level aggregation)
|
||||
|
||||
const mongoose = require('mongoose');
|
||||
|
||||
const baseOptions = {
|
||||
timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
|
||||
};
|
||||
|
||||
// ─── Bandwidth Summary (per agent, per collection cycle) ───────────────────────
|
||||
const SummarySchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true }, // null = global/all agents
|
||||
site_uuid: { type: String, index: true },
|
||||
bandwidth_down: Number,
|
||||
bandwidth_up: Number,
|
||||
active_flows: Number,
|
||||
download_speed: Number,
|
||||
upload_speed: Number,
|
||||
total_devices: Number,
|
||||
total_threats: Number,
|
||||
packet_drops: Number,
|
||||
peak_flow_rate: Number,
|
||||
cpu_usage: Number,
|
||||
memory_usage: Number,
|
||||
queue_depth: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Top Applications (per agent) ─────────────────────────────────────────────
|
||||
const AppStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
app_label: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Protocol Statistics (per agent) ──────────────────────────────────────────
|
||||
const ProtocolStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
protocol_label: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Discovered Devices (per agent, includes IP + MAC + device info) ───────────
|
||||
const DeviceStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
ip_address: { type: String, required: true, index: true },
|
||||
mac_address: { type: String, index: true },
|
||||
device_label: String,
|
||||
device_type: String,
|
||||
os_label: String,
|
||||
manufacturer: String,
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
last_seen: String,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Network Flows (per agent) ─────────────────────────────────────────────────
|
||||
const FlowSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
flow_id: String,
|
||||
src_ip: { type: String, index: true },
|
||||
src_mac: { type: String, index: true }, // indexed for MAC-to-IP resolution in events
|
||||
dst_ip: { type: String, index: true },
|
||||
dst_port: Number,
|
||||
protocol: String,
|
||||
app_label: String,
|
||||
domain: String,
|
||||
download: Number,
|
||||
upload: Number,
|
||||
first_seen: String,
|
||||
last_seen: String,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Cyber Threats (per agent) ─────────────────────────────────────────────────
|
||||
const ThreatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
threat_type: String,
|
||||
severity: String,
|
||||
src_ip: String,
|
||||
dst_ip: String,
|
||||
dst_port: Number,
|
||||
protocol: String,
|
||||
description: String,
|
||||
event_at: String,
|
||||
flow_id: { type: String, index: true },
|
||||
}, baseOptions);
|
||||
|
||||
// ─── App Categories (per agent) ───────────────────────────────────────────────
|
||||
const AppCategoryStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
category_label: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── System Events (per agent) ─────────────────────────────────────────────────
|
||||
const EventSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
event_id: Number,
|
||||
event_type: String,
|
||||
severity: String,
|
||||
description: String,
|
||||
category_label: String,
|
||||
ip_address: String,
|
||||
mac_address: String,
|
||||
event_at: Date,
|
||||
flow_id: { type: String, index: true },
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Compound indexes for common dashboard queries ─────────────────────────────
|
||||
SummarySchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
AppStatSchema.index({ agent_uuid: 1, timestamp: -1, download: -1 });
|
||||
DeviceStatSchema.index({ agent_uuid: 1, ip_address: 1 }, { unique: true });
|
||||
FlowSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
FlowSchema.index({ agent_uuid: 1, flow_id: 1 });
|
||||
FlowSchema.index({ site_uuid: 1, timestamp: -1 });
|
||||
FlowSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
|
||||
ThreatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
AppCategoryStatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
EventSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
FlowSchema.index({ site_uuid: 1, src_mac: 1, timestamp: -1 }); // for MAC-to-IP resolution
|
||||
|
||||
// ── Per-Device Per-Application Stats ────────────────────────────────────────
|
||||
// Collected from DPI API: /data/stats/top/application/download with filter_local_ips
|
||||
// Allows showing "YouTube 134GB" in Device Detail modal per specific IP
|
||||
const DeviceAppStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
ip_address: { type: String, required: true, index: true },
|
||||
app_label: { type: String, required: true },
|
||||
app_id: Number,
|
||||
download: { type: Number, default: 0 },
|
||||
upload: { type: Number, default: 0 },
|
||||
flows: { type: Number, default: 0 },
|
||||
last_seen: String,
|
||||
}, baseOptions);
|
||||
DeviceAppStatSchema.index({ agent_uuid: 1, ip_address: 1, timestamp: -1 });
|
||||
DeviceAppStatSchema.index({ ip_address: 1, app_label: 1, timestamp: -1 });
|
||||
DeviceAppStatSchema.index({ site_uuid: 1, app_label: 1, timestamp: -1 });
|
||||
|
||||
const telemetrySchemas = require('./SchemasTelemetry');
|
||||
const auxSchemas = require('./SchemasAux');
|
||||
|
||||
// ─── Agent Registry (all agents registered in Netify, regardless of activity) ──
|
||||
// Upserted every collector cycle. Source of truth for the agents list page.
|
||||
const AgentRegistrySchema = new mongoose.Schema({
|
||||
uuid: { type: String, required: true, unique: true, index: true },
|
||||
serial: { type: String },
|
||||
label: { type: String },
|
||||
site_uuid: { type: String, index: true },
|
||||
provisioned: { type: Boolean, default: false },
|
||||
activated: { type: Boolean, default: false },
|
||||
last_seen_at: { type: mongoose.Schema.Types.Mixed },
|
||||
netify_id: { type: Number },
|
||||
}, { ...baseOptions, collection: 'agent_registry' });
|
||||
|
||||
module.exports = {
|
||||
Summary: mongoose.model('Summary', SummarySchema),
|
||||
AppStat: mongoose.model('AppStat', AppStatSchema),
|
||||
ProtocolStat: mongoose.model('ProtocolStat', ProtocolStatSchema),
|
||||
DeviceStat: mongoose.model('DeviceStat', DeviceStatSchema),
|
||||
DeviceAppStat: mongoose.model('DeviceAppStat', DeviceAppStatSchema),
|
||||
Flow: mongoose.model('Flow', FlowSchema),
|
||||
Threat: mongoose.model('Threat', ThreatSchema),
|
||||
AppCategoryStat: mongoose.model('AppCategoryStat', AppCategoryStatSchema),
|
||||
Event: mongoose.model('Event', EventSchema),
|
||||
AgentRegistry: mongoose.model('AgentRegistry', AgentRegistrySchema),
|
||||
...auxSchemas,
|
||||
...telemetrySchemas,
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
// proxy/netifyAgentFetcher.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Fetches active agents from Netify Informatics API.
|
||||
// Uses a two-strategy approach to handle endpoints that may timeout (Source 2).
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const { netifyFetch, agentMap } = require('./netifyClientCore');
|
||||
|
||||
// Strategy 1 timeout: 15s (agent/download can be slow on small deployments)
|
||||
const PRIMARY_TIMEOUT_MS = 15000;
|
||||
|
||||
async function fetchAgents(siteUuid = null) {
|
||||
// Strategy 1: Use /data/stats/top/agent/download (standard Netify endpoint)
|
||||
// filter_interval reduced to 31 days to lessen query load vs. old 365-day value.
|
||||
const primaryPromise = (async () => {
|
||||
try {
|
||||
const data = await netifyFetch('/data/stats/top/agent/download', {
|
||||
filter_interval: 44640, // 31 days
|
||||
settings_limit: 1000000,
|
||||
}, null, siteUuid);
|
||||
if (data && Array.isArray(data) && data.length > 0) return data;
|
||||
return null;
|
||||
} catch (e) {
|
||||
return null;
|
||||
}
|
||||
})();
|
||||
|
||||
const timeoutPromise = new Promise(resolve =>
|
||||
setTimeout(() => resolve(null), PRIMARY_TIMEOUT_MS)
|
||||
);
|
||||
|
||||
const primaryData = await Promise.race([primaryPromise, timeoutPromise]);
|
||||
|
||||
if (primaryData && Array.isArray(primaryData) && primaryData.length > 0) {
|
||||
const list = primaryData.map(r => ({
|
||||
id: r.agent?.id,
|
||||
uuid: r.agent?.uuid || r.agent?.serial,
|
||||
serial: r.agent?.serial,
|
||||
label: r.agent?.label || r.agent?.serial,
|
||||
provisioned: true,
|
||||
activated: true,
|
||||
last_seen_at: r.agent?.last_seen_at ?? null,
|
||||
})).filter(a => a.uuid);
|
||||
|
||||
// Populate agentMap (uuid → id) for downstream filter_agents usage
|
||||
for (const a of list) {
|
||||
if (a.uuid && a.id) agentMap[a.uuid] = a.id;
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
// Strategy 2: Fallback — discover agents from /data/flows
|
||||
// Useful for Source 2 where /data/stats/top/agent/download consistently times out.
|
||||
console.log('[fetchAgents] Primary endpoint timeout/empty. Using flows-based agent discovery...');
|
||||
try {
|
||||
const flowData = await netifyFetch('/data/flows', {
|
||||
settings_limit: 100,
|
||||
}, null, siteUuid);
|
||||
|
||||
if (!flowData || !Array.isArray(flowData)) return [];
|
||||
|
||||
// Extract unique agent_uuids from flow records
|
||||
const seen = new Set();
|
||||
const agentList = [];
|
||||
for (const flow of flowData) {
|
||||
const uuid = flow.agent_uuid;
|
||||
if (uuid && !seen.has(uuid)) {
|
||||
seen.add(uuid);
|
||||
agentList.push({
|
||||
id: null,
|
||||
uuid: uuid,
|
||||
serial: uuid,
|
||||
label: uuid,
|
||||
provisioned: true,
|
||||
activated: true,
|
||||
last_seen_at: flow.last_seen_at ?? null,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[fetchAgents] Fallback discovered ${agentList.length} agent(s) from flows.`);
|
||||
return agentList;
|
||||
} catch (e) {
|
||||
console.error('[fetchAgents] Fallback also failed:', e.message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { fetchAgents };
|
||||
+16
-227
@@ -3,157 +3,11 @@
|
||||
// DPI API wrapper for the BackOne Proxy Server targeting original Netify API.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const { netifyFetch, BASE_URL, agentMap } = require('./netifyClientCore');
|
||||
const { netifyFetch, BASE_URL } = require('./netifyClientCore');
|
||||
const telemetry = require('./netifyTelemetry');
|
||||
const stats = require('./netifyClientStats');
|
||||
|
||||
const PORT_SERVICE_MAP = {
|
||||
80: 'HTTP', 443: 'HTTPS / TLS', 8080: 'HTTP Alt', 8443: 'HTTPS Alt',
|
||||
53: 'DNS', 5353: 'mDNS', 853: 'DNS-over-TLS',
|
||||
25: 'SMTP', 587: 'SMTP TLS', 465: 'SMTPS', 110: 'POP3', 143: 'IMAP',
|
||||
22: 'SSH', 23: 'Telnet', 3389: 'RDP', 5900: 'VNC',
|
||||
21: 'FTP', 20: 'FTP Data', 989: 'FTPS', 990: 'FTPS Control',
|
||||
3306: 'MySQL', 5432: 'PostgreSQL', 6379: 'Redis', 27017: 'MongoDB',
|
||||
1194: 'OpenVPN', 51820: 'WireGuard', 500: 'IPSec IKE', 4500: 'IPSec NAT-T',
|
||||
67: 'DHCP', 68: 'DHCP Client', 123: 'NTP',
|
||||
6881: 'BitTorrent', 6882: 'BitTorrent', 6883: 'BitTorrent',
|
||||
9993: 'ZeroTier VPN',
|
||||
};
|
||||
|
||||
async function fetchAgents(siteUuid = null) {
|
||||
const data = await netifyFetch('/data/stats/top/agent/download', { filter_interval: 43200, settings_limit: 100 }, null, siteUuid);
|
||||
if (!data || !Array.isArray(data)) return [];
|
||||
const list = data.map(r => ({
|
||||
id: r.agent?.id,
|
||||
uuid: r.agent?.uuid,
|
||||
label: r.agent?.label,
|
||||
})).filter(a => a.uuid);
|
||||
|
||||
for (const a of list) {
|
||||
if (a.uuid && a.id) agentMap[a.uuid] = a.id;
|
||||
}
|
||||
|
||||
// Secondary validation: if this site already has data in MongoDB, only return agents
|
||||
// that have at least one summary record for THIS site_uuid. This prevents the
|
||||
// org-level stats endpoint from cross-contaminating agents across sites.
|
||||
if (siteUuid) {
|
||||
try {
|
||||
const mongoose = require('mongoose');
|
||||
if (mongoose.connection.readyState === 1) {
|
||||
const db = mongoose.connection.db;
|
||||
const knownAgents = await db.collection('summaries').distinct('agent_uuid', {
|
||||
site_uuid: siteUuid,
|
||||
agent_uuid: { $ne: null },
|
||||
});
|
||||
|
||||
if (knownAgents.length > 0) {
|
||||
const knownSet = new Set(knownAgents);
|
||||
const validated = list.filter(a => knownSet.has(a.uuid));
|
||||
// If MongoDB cross-check yields results, use the validated list.
|
||||
// On first boot (no DB data yet), fall through and use the full API list.
|
||||
if (validated.length > 0) return validated;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('[Collector] fetchAgents DB cross-check failed:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
|
||||
async function fetchBandwidthSummary(interval = 1440, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData, flowsData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/local_ip/download', { filter_interval: interval, settings_limit: 500 }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/local_ip/upload', { filter_interval: interval, settings_limit: 500 }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/local_ip/flow_count', { filter_interval: interval, settings_limit: 500 }, agentUuid, siteUuid),
|
||||
]);
|
||||
const bandwidth_down = dlData?.reduce((s, r) => s + (r.download ?? 0), 0) ?? 0;
|
||||
const bandwidth_up = ulData?.reduce((s, r) => s + (r.upload ?? 0), 0) ?? 0;
|
||||
const total_devices = dlData?.length ?? 0;
|
||||
const active_flows = flowsData?.reduce((s, r) => s + (r.flows ?? r.flow_count ?? 0), 0) ?? 0;
|
||||
return { bandwidth_down, bandwidth_up, total_devices, active_flows, total_threats: 0 };
|
||||
}
|
||||
|
||||
async function fetchTopApps(interval = 1440, limit = 200, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/application/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/application/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return null;
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const id = r.application?.id;
|
||||
if (id) ulMap[id] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => ({
|
||||
application: {
|
||||
id: r.application?.id ?? null,
|
||||
label: r.application?.label ?? 'Unknown',
|
||||
tag: r.application?.tag ?? null,
|
||||
},
|
||||
download: r.download ?? 0,
|
||||
upload: ulMap[r.application?.id] ?? 0,
|
||||
flows: r.flows ?? 0,
|
||||
}));
|
||||
}
|
||||
|
||||
async function fetchDiscoveredDevices(interval = 1440, limit = 500, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/local_ip/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/local_ip/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return null;
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const ip = r.local_ip?.address ?? String(r.local_ip);
|
||||
ulMap[ip] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
const ip = r.local_ip?.address ?? String(r.local_ip ?? '');
|
||||
return {
|
||||
ip_address: ip,
|
||||
mac_address: r.local_mac ?? null,
|
||||
device_label: r.device_label ?? ip,
|
||||
device_type: r.device_type ?? null,
|
||||
os_label: r.os_label ?? null,
|
||||
manufacturer: r.manufacturer ?? null,
|
||||
download: r.download ?? 0,
|
||||
upload: ulMap[ip] ?? 0,
|
||||
flows: r.flows ?? 0,
|
||||
last_seen: r.last_seen_at?.date ?? null,
|
||||
};
|
||||
}).filter(d => d.ip_address);
|
||||
}
|
||||
|
||||
async function fetchDeviceApps(ipAddress, interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
const ipFilter = JSON.stringify([ipAddress]);
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/application/download', { filter_interval: interval, settings_limit: limit, filter_local_ips: ipFilter }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/application/upload', { filter_interval: interval, settings_limit: limit, filter_local_ips: ipFilter }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData || !Array.isArray(dlData)) return [];
|
||||
const ulMap = {};
|
||||
if (ulData && Array.isArray(ulData)) {
|
||||
for (const r of ulData) {
|
||||
const id = r.application?.id;
|
||||
if (id) ulMap[id] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => ({
|
||||
app_label: r.application?.label ?? 'Unknown',
|
||||
app_id: r.application?.id ?? null,
|
||||
download: r.download ?? 0,
|
||||
upload: ulMap[r.application?.id] ?? 0,
|
||||
flows: r.flows ?? 0,
|
||||
})).filter(a => a.download > 0 || a.upload > 0);
|
||||
}
|
||||
|
||||
async function fetchFlows(limit = 10000, agentUuid = null, siteUuid = null) {
|
||||
async function fetchFlows(limit = 1000000, agentUuid = null, siteUuid = null) {
|
||||
const [raw, sniRaw] = await Promise.all([
|
||||
netifyFetch('/data/flows', { settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/tls_sni/download', { filter_interval: 1440, settings_limit: 50 }, agentUuid, siteUuid),
|
||||
@@ -163,12 +17,14 @@ async function fetchFlows(limit = 10000, agentUuid = null, siteUuid = null) {
|
||||
if (sniRaw && Array.isArray(sniRaw)) {
|
||||
for (const r of sniRaw) {
|
||||
const sni = typeof r.tls_sni === 'object' ? r.tls_sni?.label : r.tls_sni;
|
||||
if (sni && typeof sni === 'string' && sni.trim() !== '') sniList.push(sni.replace(/^\*\./, '').trim());
|
||||
if (sni && typeof sni === 'string' && sni.trim() !== '') {
|
||||
sniList.push(sni.replace(/^\*\./, '').trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
return raw.map(r => {
|
||||
const port = r.remote_port ?? null;
|
||||
const portService = port ? (PORT_SERVICE_MAP[port] ?? `Port ${port}`) : null;
|
||||
const portService = port ? (stats.PORT_SERVICE_MAP[port] ?? `Port ${port}`) : null;
|
||||
const appLabel = r.application?.label || portService;
|
||||
const domain = r.tls_sni || r.dns_hostname || r.hostname || null;
|
||||
return {
|
||||
@@ -188,84 +44,17 @@ async function fetchFlows(limit = 10000, agentUuid = null, siteUuid = null) {
|
||||
}).filter(f => f.src_ip);
|
||||
}
|
||||
|
||||
async function fetchCyberThreats(agentUuid = null, siteUuid = null) {
|
||||
const ipRepData = await netifyFetch('/data/stats/top/remote_ip/download', { filter_interval: 1440, settings_limit: 50 }, agentUuid, siteUuid);
|
||||
if (!ipRepData || !Array.isArray(ipRepData)) return [];
|
||||
const SUSPICIOUS_PORTS = new Set([23, 4444, 1337, 6667, 31337, 12345, 54321, 4899, 5554, 9999]);
|
||||
const threats = [];
|
||||
for (const r of ipRepData) {
|
||||
const ip = r.remote_ip?.address ?? null;
|
||||
const port = r.remote_port ?? 0;
|
||||
if (ip && SUSPICIOUS_PORTS.has(port)) {
|
||||
threats.push({
|
||||
threat_type: `Suspicious Port ${port}`,
|
||||
severity: 'High',
|
||||
src_ip: null,
|
||||
dst_ip: ip,
|
||||
dst_port: port,
|
||||
protocol: r.ip_protocol?.label ?? null,
|
||||
description: `Suspicious outbound connection to ${ip}:${port}`,
|
||||
event_at: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
}
|
||||
return threats;
|
||||
}
|
||||
|
||||
async function fetchEvents(limit = 100, agentUuid = null, siteUuid = null) {
|
||||
const raw = await netifyFetch('/event/events', { settings_limit: limit }, agentUuid, siteUuid);
|
||||
if (!raw || !Array.isArray(raw)) return [];
|
||||
return raw.map(r => {
|
||||
let msg = r.label || '';
|
||||
if (r.description) {
|
||||
try {
|
||||
const descObj = JSON.parse(r.description);
|
||||
msg = descObj.default || r.label || '';
|
||||
if (descObj.tags) {
|
||||
for (const k in descObj.tags) {
|
||||
const tagVal = descObj.tags[k];
|
||||
const val = Array.isArray(tagVal) ? (tagVal[0] === 'Unknown' && tagVal[1] ? tagVal[1] : tagVal[0]) : tagVal;
|
||||
msg = msg.replace(`{{ ${k} }}`, val).replace(`{{${k}}}`, val);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
msg = r.description;
|
||||
}
|
||||
}
|
||||
let sevLabel = 'Info';
|
||||
if (r.severity >= 30) sevLabel = 'Critical';
|
||||
else if (r.severity >= 20) sevLabel = 'High';
|
||||
else if (r.severity >= 10) sevLabel = 'Warning';
|
||||
let srcIp = null;
|
||||
if (r.description) {
|
||||
try {
|
||||
const descObj = JSON.parse(r.description);
|
||||
srcIp = descObj.tags?.device_ip || descObj.tags?.ip || null;
|
||||
} catch {}
|
||||
}
|
||||
return {
|
||||
event_id: r.id || null,
|
||||
event_type: r.basename || 'unknown',
|
||||
severity: sevLabel,
|
||||
description: msg,
|
||||
category_label: r.category?.label || 'Intelligence',
|
||||
ip_address: srcIp,
|
||||
mac_address: r.additional?.device?.mac?.address || null,
|
||||
event_at: r.created_at?.date ? new Date(r.created_at.date) : new Date()
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchAgents,
|
||||
fetchBandwidthSummary,
|
||||
fetchTopApps,
|
||||
fetchDiscoveredDevices,
|
||||
fetchDeviceApps,
|
||||
fetchFlows,
|
||||
fetchCyberThreats,
|
||||
fetchEvents,
|
||||
fetchAgents: stats.fetchAgents,
|
||||
fetchBandwidthSummary: stats.fetchBandwidthSummary,
|
||||
fetchTopApps: stats.fetchTopApps,
|
||||
fetchDiscoveredDevices: stats.fetchDiscoveredDevices,
|
||||
fetchDeviceApps: stats.fetchDeviceApps,
|
||||
fetchCyberThreats: stats.fetchCyberThreats,
|
||||
fetchEvents: stats.fetchEvents,
|
||||
syncApplicationDictionary: stats.syncApplicationDictionary,
|
||||
BASE_URL,
|
||||
PORT_SERVICE_MAP,
|
||||
PORT_SERVICE_MAP: stats.PORT_SERVICE_MAP,
|
||||
...telemetry,
|
||||
};
|
||||
@@ -0,0 +1,219 @@
|
||||
// proxy/netifyClientStats.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Supplementary fetchers split from netifyClient.js to satisfy the 256-line limit.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const { netifyFetch } = require('./netifyClientCore');
|
||||
const { fetchAgents } = require('./netifyAgentFetcher');
|
||||
|
||||
const PORT_SERVICE_MAP = {
|
||||
80: 'HTTP', 443: 'HTTPS / TLS', 8080: 'HTTP Alt', 8443: 'HTTPS Alt',
|
||||
53: 'DNS', 5353: 'mDNS', 853: 'DNS-over-TLS',
|
||||
25: 'SMTP', 587: 'SMTP TLS', 465: 'SMTPS', 110: 'POP3', 143: 'IMAP',
|
||||
22: 'SSH', 23: 'Telnet', 3389: 'RDP', 5900: 'VNC',
|
||||
21: 'FTP', 20: 'FTP Data', 989: 'FTPS', 990: 'FTPS Control',
|
||||
3306: 'MySQL', 5432: 'PostgreSQL', 6379: 'Redis', 27017: 'MongoDB',
|
||||
1194: 'OpenVPN', 51820: 'WireGuard', 500: 'IPSec IKE', 4500: 'IPSec NAT-T',
|
||||
67: 'DHCP', 68: 'DHCP Client', 123: 'NTP',
|
||||
6881: 'BitTorrent', 6882: 'BitTorrent', 6883: 'BitTorrent',
|
||||
9993: 'ZeroTier VPN',
|
||||
};
|
||||
|
||||
async function fetchBandwidthSummary(interval = 1440, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData, flowsData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/local_ip/download', { filter_interval: interval, settings_limit: 500 }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/local_ip/upload', { filter_interval: interval, settings_limit: 500 }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/local_ip/flow_count', { filter_interval: interval, settings_limit: 500 }, agentUuid, siteUuid),
|
||||
]);
|
||||
const bandwidth_down = dlData?.reduce((s, r) => s + (r.download ?? 0), 0) ?? 0;
|
||||
const bandwidth_up = ulData?.reduce((s, r) => s + (r.upload ?? 0), 0) ?? 0;
|
||||
const total_devices = dlData?.length ?? 0;
|
||||
const active_flows = flowsData?.reduce((s, r) => s + (r.flows ?? r.flow_count ?? 0), 0) ?? 0;
|
||||
return { bandwidth_down, bandwidth_up, total_devices, active_flows, total_threats: 0 };
|
||||
}
|
||||
|
||||
async function fetchTopApps(interval = 1440, limit = 200, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/application/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/application/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return null;
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const id = r.application?.id;
|
||||
if (id) ulMap[id] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => ({
|
||||
application: { id: r.application?.id ?? null, label: r.application?.label ?? 'Unknown', tag: r.application?.tag ?? null },
|
||||
download: r.download ?? 0, upload: ulMap[r.application?.id] ?? 0, flows: r.flows ?? 0,
|
||||
}));
|
||||
}
|
||||
|
||||
async function fetchDiscoveredDevices(interval = 1440, limit = 500, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/local_ip/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/local_ip/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return null;
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const ip = r.local_ip?.address ?? String(r.local_ip);
|
||||
ulMap[ip] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
const ip = r.local_ip?.address ?? String(r.local_ip ?? '');
|
||||
return {
|
||||
ip_address: ip, mac_address: r.local_mac ?? null, device_label: r.device_label ?? ip, device_type: r.device_type ?? null,
|
||||
os_label: r.os_label ?? null, manufacturer: r.manufacturer ?? null, download: r.download ?? 0, upload: ulMap[ip] ?? 0,
|
||||
flows: r.flows ?? 0, last_seen: r.last_seen_at?.date ?? null,
|
||||
};
|
||||
}).filter(d => d.ip_address);
|
||||
}
|
||||
|
||||
async function fetchDeviceApps(ipAddress, interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
const ipFilter = JSON.stringify([ipAddress]);
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/application/download', { filter_interval: interval, settings_limit: limit, filter_local_ips: ipFilter }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/application/upload', { filter_interval: interval, settings_limit: limit, filter_local_ips: ipFilter }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData || !Array.isArray(dlData)) return [];
|
||||
const ulMap = {};
|
||||
if (ulData && Array.isArray(ulData)) {
|
||||
for (const r of ulData) {
|
||||
const id = r.application?.id;
|
||||
if (id) ulMap[id] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => ({
|
||||
app_label: r.application?.label ?? 'Unknown',
|
||||
app_id: r.application?.id ?? null,
|
||||
download: r.download ?? 0,
|
||||
upload: ulMap[r.application?.id] ?? 0,
|
||||
flows: r.flows ?? 0,
|
||||
})).filter(a => a.download > 0 || a.upload > 0);
|
||||
}
|
||||
|
||||
async function fetchCyberThreats(agentUuid = null, siteUuid = null) {
|
||||
const ipRepData = await netifyFetch('/data/stats/top/remote_ip/download', { filter_interval: 1440, settings_limit: 50 }, agentUuid, siteUuid);
|
||||
if (!ipRepData || !Array.isArray(ipRepData)) return [];
|
||||
const SUSPICIOUS_PORTS = new Set([23, 4444, 1337, 6667, 31337, 12345, 54321, 4899, 5554, 9999]);
|
||||
const threats = [];
|
||||
for (const r of ipRepData) {
|
||||
const ip = r.remote_ip?.address ?? null;
|
||||
const port = r.remote_port ?? 0;
|
||||
if (ip && SUSPICIOUS_PORTS.has(port)) {
|
||||
threats.push({
|
||||
threat_type: `Suspicious Port ${port}`,
|
||||
severity: 'High',
|
||||
src_ip: null,
|
||||
dst_ip: ip,
|
||||
dst_port: port,
|
||||
protocol: r.ip_protocol?.label ?? null,
|
||||
description: `Suspicious outbound connection to ${ip}:${port}`,
|
||||
event_at: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
}
|
||||
return threats;
|
||||
}
|
||||
|
||||
async function fetchEvents(limit = 100, agentUuid = null, siteUuid = null) {
|
||||
const raw = await netifyFetch('/event/events', { settings_limit: limit }, agentUuid, siteUuid);
|
||||
if (!raw || !Array.isArray(raw)) return [];
|
||||
return raw.map(r => {
|
||||
let msg = r.label || '';
|
||||
if (r.description) {
|
||||
try {
|
||||
const descObj = JSON.parse(r.description);
|
||||
msg = descObj.default || r.label || '';
|
||||
if (descObj.tags) {
|
||||
for (const k in descObj.tags) {
|
||||
const tagVal = descObj.tags[k];
|
||||
const val = Array.isArray(tagVal) ? (tagVal[0] === 'Unknown' && tagVal[1] ? tagVal[1] : tagVal[0]) : tagVal;
|
||||
msg = msg.replace(`{{ ${k} }}`, val).replace(`{{${k}}}`, val);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
msg = r.description;
|
||||
}
|
||||
}
|
||||
let sevLabel = 'Info';
|
||||
if (r.severity >= 30) sevLabel = 'Critical';
|
||||
else if (r.severity >= 20) sevLabel = 'High';
|
||||
else if (r.severity >= 10) sevLabel = 'Warning';
|
||||
let srcIp = null;
|
||||
if (r.description) {
|
||||
try {
|
||||
const descObj = JSON.parse(r.description);
|
||||
srcIp = descObj.tags?.device_ip || descObj.tags?.ip || null;
|
||||
} catch {}
|
||||
}
|
||||
return {
|
||||
event_id: r.id || null,
|
||||
event_type: r.basename || 'unknown',
|
||||
severity: sevLabel,
|
||||
description: msg,
|
||||
category_label: r.category?.label || 'Intelligence',
|
||||
ip_address: srcIp,
|
||||
mac_address: r.additional?.device?.mac?.address || null,
|
||||
event_at: r.created_at?.date ? new Date(r.created_at.date) : new Date()
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function syncApplicationDictionary() {
|
||||
const mongoose = require('mongoose');
|
||||
const { LookupApp } = require('./models/Schemas');
|
||||
|
||||
console.log('[Netify] Fetching application catalog...');
|
||||
const allApps = await netifyFetch('/lookup/applications', { settings_limit: 5000 });
|
||||
if (!allApps || !Array.isArray(allApps)) {
|
||||
console.error('[Netify] Failed to fetch application dictionary.');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[Netify] Application catalog fetched successfully. Got ${allApps.length} apps.`);
|
||||
if (allApps.length === 0) return;
|
||||
|
||||
console.log(`[Netify] Syncing ${allApps.length} application definitions to MongoDB...`);
|
||||
await LookupApp.deleteMany({});
|
||||
|
||||
const batchSize = 100;
|
||||
for (let i = 0; i < allApps.length; i += batchSize) {
|
||||
const batch = allApps.slice(i, i + batchSize);
|
||||
await LookupApp.insertMany(batch.map(app => ({
|
||||
id: app.id,
|
||||
name: app.name,
|
||||
label: app.label,
|
||||
tag: app.tag,
|
||||
description: app.description,
|
||||
full_name: app.full_name || app.application?.full_label || null,
|
||||
favicon: app.favicon || app.application?.favicon || null,
|
||||
icon: app.icon || app.application?.icon || null,
|
||||
logo: app.logo || app.application?.logo || null,
|
||||
application_category: {
|
||||
id: app.application_category?.id,
|
||||
name: app.application_category?.name,
|
||||
label: app.application_category?.label,
|
||||
tag: app.application_category?.tag
|
||||
}
|
||||
})));
|
||||
}
|
||||
console.log('[Netify] ✓ Application dictionary sync completed.');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchAgents,
|
||||
fetchBandwidthSummary,
|
||||
fetchTopApps,
|
||||
fetchDiscoveredDevices,
|
||||
fetchDeviceApps,
|
||||
fetchCyberThreats,
|
||||
fetchEvents,
|
||||
syncApplicationDictionary,
|
||||
PORT_SERVICE_MAP
|
||||
};
|
||||
+234
-234
@@ -1,234 +1,234 @@
|
||||
// proxy/netifyTelemetry.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Supplementary Telemetry endpoints wrapper for BackOne Proxy Server
|
||||
// Split from netifyClient.js to strictly respect the 256-line file size limit.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const { netifyFetch } = require('./netifyClientCore');
|
||||
|
||||
async function fetchTopAppCategories(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/application_category/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/application_category/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return [];
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const key = r.application_category?.label ?? r.application_category;
|
||||
if (key) ulMap[key] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
const label = r.application_category?.label ?? String(r.application_category ?? 'Unknown');
|
||||
return {
|
||||
category_label : label,
|
||||
download : r.download ?? 0,
|
||||
upload : ulMap[label] ?? 0,
|
||||
flows : r.flows ?? 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchTlsVersions(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/tls_version/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/tls_version/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return [];
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const key = r.tls_version?.label ?? r.tls_version?.code ?? r.tls_version;
|
||||
if (key) ulMap[key] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
const label = r.tls_version?.label ?? r.tls_version?.code ?? String(r.tls_version ?? 'Unknown');
|
||||
return {
|
||||
tls_version : label,
|
||||
download : r.download ?? 0,
|
||||
upload : ulMap[label] ?? 0,
|
||||
flows : r.flows ?? 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchTlsCiphers(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/tls_cipher/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/tls_cipher/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return [];
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const key = r.tls_cipher?.label ?? r.tls_cipher?.code ?? r.tls_cipher;
|
||||
if (key) ulMap[key] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
const label = r.tls_cipher?.label ?? r.tls_cipher?.code ?? String(r.tls_cipher ?? 'Unknown');
|
||||
return {
|
||||
tls_cipher : label,
|
||||
download : r.download ?? 0,
|
||||
upload : ulMap[label] ?? 0,
|
||||
flows : r.flows ?? 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchTlsSecurity(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/tls_security/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/tls_security/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return [];
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const key = r.tls_security?.label ?? r.tls_security?.code ?? r.tls_security;
|
||||
if (key) ulMap[key] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
const label = r.tls_security?.label ?? r.tls_security?.code ?? String(r.tls_security ?? 'Unknown');
|
||||
return {
|
||||
tls_security : label,
|
||||
download : r.download ?? 0,
|
||||
upload : ulMap[label] ?? 0,
|
||||
flows : r.flows ?? 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchTopCountries(interval = 1440, limit = 100, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/country/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/country/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return [];
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const cc = r.country?.code;
|
||||
if (cc) ulMap[cc] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
return {
|
||||
country_code: r.country?.code ?? 'Unknown',
|
||||
country_name: r.country?.label ?? '',
|
||||
download: r.download ?? 0,
|
||||
upload: ulMap[r.country?.code] ?? 0,
|
||||
flows: r.flows ?? 0,
|
||||
};
|
||||
}).filter(r => r.country_code);
|
||||
}
|
||||
|
||||
async function fetchTopProperty(fieldName, interval, limit, agentUuid, siteUuid) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch(`/data/stats/top/${fieldName}/download`, { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch(`/data/stats/top/${fieldName}/upload`, { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return [];
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const item = r[fieldName];
|
||||
const key = item?.hash ?? item?.name ?? item?.label ?? String(item ?? '');
|
||||
if (key) ulMap[key] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
const item = r[fieldName];
|
||||
const key = item?.hash ?? item?.name ?? item?.label ?? String(item || 'Unknown');
|
||||
const label = item?.label ?? key;
|
||||
return {
|
||||
key,
|
||||
label,
|
||||
download: r.download ?? 0,
|
||||
upload: ulMap[key] ?? ulMap[label] ?? 0,
|
||||
flows: r.flows ?? 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function mapProp(data, keyName) {
|
||||
return data.map(d => ({
|
||||
[keyName]: d.key,
|
||||
download: d.download,
|
||||
upload: d.upload,
|
||||
flows: d.flows,
|
||||
}));
|
||||
}
|
||||
|
||||
async function fetchDhcpFingerprints(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('dhcp_class', interval, limit, agentUuid, siteUuid), 'fingerprint');
|
||||
}
|
||||
|
||||
async function fetchHttpUserAgents(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('http_useragent', interval, limit, agentUuid, siteUuid), 'user_agent');
|
||||
}
|
||||
|
||||
async function fetchBittorrentHashes(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
const data = await fetchTopProperty('bittorrent_info_hash', interval, limit, agentUuid, siteUuid);
|
||||
return data.map(d => ({
|
||||
info_hash: d.key,
|
||||
label: d.label,
|
||||
download: d.download,
|
||||
upload: d.upload,
|
||||
flows: d.flows,
|
||||
}));
|
||||
}
|
||||
|
||||
async function fetchSniHostnames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('tls_sni', interval, limit, agentUuid, siteUuid), 'sni_hostname');
|
||||
}
|
||||
|
||||
async function fetchSslServerCn(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('ssl_server_cn', interval, limit, agentUuid, siteUuid), 'ssl_server_cn');
|
||||
}
|
||||
|
||||
async function fetchQuicHostnames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('quic_hostname', interval, limit, agentUuid, siteUuid), 'quic_hostname');
|
||||
}
|
||||
|
||||
async function fetchSshClients(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('ssh_client', interval, limit, agentUuid, siteUuid), 'ssh_client');
|
||||
}
|
||||
|
||||
async function fetchSshServers(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('ssh_server', interval, limit, agentUuid, siteUuid), 'ssh_server');
|
||||
}
|
||||
|
||||
async function fetchMdnsHostnames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('mdns_hostname', interval, limit, agentUuid, siteUuid), 'mdns_hostname');
|
||||
}
|
||||
|
||||
async function fetchTopProtocols(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('ip_protocol', interval, limit, agentUuid, siteUuid), 'protocol_label');
|
||||
}
|
||||
|
||||
async function fetchSslSubjectAltNames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('ssl_subject_alt_name', interval, limit, agentUuid, siteUuid), 'alt_name');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchTopAppCategories,
|
||||
fetchTlsVersions,
|
||||
fetchTlsCiphers,
|
||||
fetchTlsSecurity,
|
||||
fetchTopCountries,
|
||||
fetchDhcpFingerprints,
|
||||
fetchHttpUserAgents,
|
||||
fetchBittorrentHashes,
|
||||
fetchSniHostnames,
|
||||
fetchSslServerCn,
|
||||
fetchQuicHostnames,
|
||||
fetchSshClients,
|
||||
fetchSshServers,
|
||||
fetchMdnsHostnames,
|
||||
fetchTopProtocols,
|
||||
fetchSslSubjectAltNames,
|
||||
};
|
||||
// proxy/netifyTelemetry.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Supplementary Telemetry endpoints wrapper for BackOne Proxy Server
|
||||
// Split from netifyClient.js to strictly respect the 256-line file size limit.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const { netifyFetch } = require('./netifyClientCore');
|
||||
|
||||
async function fetchTopAppCategories(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/application_category/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/application_category/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return [];
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const key = r.application_category?.label ?? r.application_category;
|
||||
if (key) ulMap[key] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
const label = r.application_category?.label ?? String(r.application_category ?? 'Unknown');
|
||||
return {
|
||||
category_label : label,
|
||||
download : r.download ?? 0,
|
||||
upload : ulMap[label] ?? 0,
|
||||
flows : r.flows ?? 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchTlsVersions(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/tls_version/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/tls_version/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return [];
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const key = r.tls_version?.label ?? r.tls_version?.code ?? r.tls_version;
|
||||
if (key) ulMap[key] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
const label = r.tls_version?.label ?? r.tls_version?.code ?? String(r.tls_version ?? 'Unknown');
|
||||
return {
|
||||
tls_version : label,
|
||||
download : r.download ?? 0,
|
||||
upload : ulMap[label] ?? 0,
|
||||
flows : r.flows ?? 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchTlsCiphers(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/tls_cipher/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/tls_cipher/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return [];
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const key = r.tls_cipher?.label ?? r.tls_cipher?.code ?? r.tls_cipher;
|
||||
if (key) ulMap[key] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
const label = r.tls_cipher?.label ?? r.tls_cipher?.code ?? String(r.tls_cipher ?? 'Unknown');
|
||||
return {
|
||||
tls_cipher : label,
|
||||
download : r.download ?? 0,
|
||||
upload : ulMap[label] ?? 0,
|
||||
flows : r.flows ?? 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchTlsSecurity(interval = 1440, limit = 15, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/tls_security/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/tls_security/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return [];
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const key = r.tls_security?.label ?? r.tls_security?.code ?? r.tls_security;
|
||||
if (key) ulMap[key] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
const label = r.tls_security?.label ?? r.tls_security?.code ?? String(r.tls_security ?? 'Unknown');
|
||||
return {
|
||||
tls_security : label,
|
||||
download : r.download ?? 0,
|
||||
upload : ulMap[label] ?? 0,
|
||||
flows : r.flows ?? 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchTopCountries(interval = 1440, limit = 100, agentUuid = null, siteUuid = null) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch('/data/stats/top/country/download', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch('/data/stats/top/country/upload', { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return [];
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const cc = r.country?.code;
|
||||
if (cc) ulMap[cc] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
return {
|
||||
country_code: r.country?.code ?? 'Unknown',
|
||||
country_name: r.country?.label ?? '',
|
||||
download: r.download ?? 0,
|
||||
upload: ulMap[r.country?.code] ?? 0,
|
||||
flows: r.flows ?? 0,
|
||||
};
|
||||
}).filter(r => r.country_code);
|
||||
}
|
||||
|
||||
async function fetchTopProperty(fieldName, interval, limit, agentUuid, siteUuid) {
|
||||
const [dlData, ulData] = await Promise.all([
|
||||
netifyFetch(`/data/stats/top/${fieldName}/download`, { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
netifyFetch(`/data/stats/top/${fieldName}/upload`, { filter_interval: interval, settings_limit: limit }, agentUuid, siteUuid),
|
||||
]);
|
||||
if (!dlData) return [];
|
||||
const ulMap = {};
|
||||
if (ulData) {
|
||||
for (const r of ulData) {
|
||||
const item = r[fieldName];
|
||||
const key = item?.hash ?? item?.name ?? item?.label ?? String(item ?? '');
|
||||
if (key) ulMap[key] = r.upload ?? 0;
|
||||
}
|
||||
}
|
||||
return dlData.map(r => {
|
||||
const item = r[fieldName];
|
||||
const key = item?.hash ?? item?.name ?? item?.label ?? String(item || 'Unknown');
|
||||
const label = item?.label ?? key;
|
||||
return {
|
||||
key,
|
||||
label,
|
||||
download: r.download ?? 0,
|
||||
upload: ulMap[key] ?? ulMap[label] ?? 0,
|
||||
flows: r.flows ?? 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function mapProp(data, keyName) {
|
||||
return data.map(d => ({
|
||||
[keyName]: d.key,
|
||||
download: d.download,
|
||||
upload: d.upload,
|
||||
flows: d.flows,
|
||||
}));
|
||||
}
|
||||
|
||||
async function fetchDhcpFingerprints(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('dhcp_class', interval, limit, agentUuid, siteUuid), 'fingerprint');
|
||||
}
|
||||
|
||||
async function fetchHttpUserAgents(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('http_useragent', interval, limit, agentUuid, siteUuid), 'user_agent');
|
||||
}
|
||||
|
||||
async function fetchBittorrentHashes(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
const data = await fetchTopProperty('bittorrent_info_hash', interval, limit, agentUuid, siteUuid);
|
||||
return data.map(d => ({
|
||||
info_hash: d.key,
|
||||
label: d.label,
|
||||
download: d.download,
|
||||
upload: d.upload,
|
||||
flows: d.flows,
|
||||
}));
|
||||
}
|
||||
|
||||
async function fetchSniHostnames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('tls_sni', interval, limit, agentUuid, siteUuid), 'sni_hostname');
|
||||
}
|
||||
|
||||
async function fetchSslServerCn(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('ssl_server_cn', interval, limit, agentUuid, siteUuid), 'ssl_server_cn');
|
||||
}
|
||||
|
||||
async function fetchQuicHostnames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('quic_hostname', interval, limit, agentUuid, siteUuid), 'quic_hostname');
|
||||
}
|
||||
|
||||
async function fetchSshClients(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('ssh_client', interval, limit, agentUuid, siteUuid), 'ssh_client');
|
||||
}
|
||||
|
||||
async function fetchSshServers(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('ssh_server', interval, limit, agentUuid, siteUuid), 'ssh_server');
|
||||
}
|
||||
|
||||
async function fetchMdnsHostnames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('mdns_hostname', interval, limit, agentUuid, siteUuid), 'mdns_hostname');
|
||||
}
|
||||
|
||||
async function fetchTopProtocols(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('ip_protocol', interval, limit, agentUuid, siteUuid), 'protocol_label');
|
||||
}
|
||||
|
||||
async function fetchSslSubjectAltNames(interval = 1440, limit = 50, agentUuid = null, siteUuid = null) {
|
||||
return mapProp(await fetchTopProperty('ssl_subject_alt_name', interval, limit, agentUuid, siteUuid), 'alt_name');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchTopAppCategories,
|
||||
fetchTlsVersions,
|
||||
fetchTlsCiphers,
|
||||
fetchTlsSecurity,
|
||||
fetchTopCountries,
|
||||
fetchDhcpFingerprints,
|
||||
fetchHttpUserAgents,
|
||||
fetchBittorrentHashes,
|
||||
fetchSniHostnames,
|
||||
fetchSslServerCn,
|
||||
fetchQuicHostnames,
|
||||
fetchSshClients,
|
||||
fetchSshServers,
|
||||
fetchMdnsHostnames,
|
||||
fetchTopProtocols,
|
||||
fetchSslSubjectAltNames,
|
||||
};
|
||||
Generated
+1312
-1312
File diff suppressed because it is too large.
Load diff
+19
-19
@@ -1,19 +1,19 @@
|
||||
{
|
||||
"name": "backone-proxy",
|
||||
"version": "1.0.0",
|
||||
"description": "BackOne DPI Proxy Server - Fetches from DPI API, filters per agent_uuid, stores to MongoDB",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"start": "node index.js",
|
||||
"start:bun": "bun run index.js",
|
||||
"dev": "nodemon index.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"axios": "^1.6.2",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^16.3.1",
|
||||
"express": "^4.18.2",
|
||||
"mongoose": "^8.0.3",
|
||||
"node-cron": "^3.0.3"
|
||||
}
|
||||
}
|
||||
{
|
||||
"name": "backone-proxy",
|
||||
"version": "1.0.0",
|
||||
"description": "BackOne DPI Proxy Server - Fetches from DPI API, filters per agent_uuid, stores to MongoDB",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"start": "node index.js",
|
||||
"start:bun": "bun run index.js",
|
||||
"dev": "nodemon index.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"axios": "^1.6.2",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^16.3.1",
|
||||
"express": "^4.18.2",
|
||||
"mongoose": "^8.0.3",
|
||||
"node-cron": "^3.0.3"
|
||||
}
|
||||
}
|
||||
+129
-124
@@ -1,124 +1,129 @@
|
||||
// proxy/scheduler.js
|
||||
// Cron scheduler for automatic data collection from DPI API
|
||||
// Runs every 5 minutes, collecting data for all agents or a specific agent.
|
||||
|
||||
const cron = require('node-cron');
|
||||
const { collectAllAgents, collectSpecificAgent, collectSpecificAgents } = require('./collector');
|
||||
|
||||
// Mode: 'all' = collect all agents, 'agent' = collect one specific agent, 'agents' = collect multiple agents
|
||||
const COLLECT_MODE = process.env.PROXY_COLLECT_MODE || 'all';
|
||||
const SPECIFIC_AGENT = process.env.PROXY_AGENT_UUID || null;
|
||||
const SPECIFIC_AGENTS = (process.env.PROXY_AGENT_UUIDS || '').split(',').map(s => s.trim()).filter(Boolean);
|
||||
const AGENT_DELAY_MS = parseInt(process.env.PROXY_AGENT_DELAY_MS || '5000');
|
||||
const CRON_SCHEDULE = process.env.PROXY_CRON_SCHEDULE || '*/5 * * * *';
|
||||
|
||||
// Capacity logging is an expensive full-scan aggregation. Run it at most once per
|
||||
// interval (default 24h) instead of every collection cycle to reduce CPU/DB load.
|
||||
const CAPACITY_LOG_INTERVAL_MS = parseInt(process.env.PROXY_CAPACITY_LOG_INTERVAL_MS || String(24 * 60 * 60 * 1000));
|
||||
|
||||
let isRunning = false;
|
||||
let lastRunAt = null;
|
||||
let lastRunResult = null;
|
||||
let runCount = 0;
|
||||
let lastCapacityLogAt = 0;
|
||||
|
||||
/**
|
||||
* Execute one collection cycle (called by cron and manual trigger).
|
||||
* Prevents concurrent runs with isRunning guard.
|
||||
*/
|
||||
async function runCollection() {
|
||||
if (isRunning) {
|
||||
console.log('[Scheduler] Skipping - previous run still in progress');
|
||||
return { skipped: true, reason: 'already_running' };
|
||||
}
|
||||
|
||||
isRunning = true;
|
||||
lastRunAt = new Date();
|
||||
runCount++;
|
||||
|
||||
try {
|
||||
let result;
|
||||
if (COLLECT_MODE === 'agent' && SPECIFIC_AGENT) {
|
||||
console.log(`[Scheduler] Run #${runCount} - Mode: SPECIFIC AGENT (${SPECIFIC_AGENT})`);
|
||||
result = await collectSpecificAgent(SPECIFIC_AGENT);
|
||||
} else if (COLLECT_MODE === 'agents' && SPECIFIC_AGENTS.length > 0) {
|
||||
console.log(`[Scheduler] Run #${runCount} - Mode: SPECIFIC AGENTS (${SPECIFIC_AGENTS.join(', ')})`);
|
||||
result = await collectSpecificAgents(SPECIFIC_AGENTS, AGENT_DELAY_MS);
|
||||
} else {
|
||||
console.log(`[Scheduler] Run #${runCount} - Mode: ALL AGENTS`);
|
||||
result = await collectAllAgents();
|
||||
}
|
||||
lastRunResult = { ...result, run_count: runCount };
|
||||
|
||||
// Log MongoDB database capacity usage (expensive full-scan aggregation).
|
||||
// Only run periodically (default: every 24h) to avoid high CPU/DB load each cycle.
|
||||
const now = Date.now();
|
||||
if (now - lastCapacityLogAt >= CAPACITY_LOG_INTERVAL_MS) {
|
||||
lastCapacityLogAt = now;
|
||||
const { logCapacityStats } = require('./db/capacityTracker');
|
||||
await logCapacityStats(`[PROXY] [MongoDB] Capacity Used after Run #${runCount}:`);
|
||||
}
|
||||
|
||||
return lastRunResult;
|
||||
} catch (err) {
|
||||
console.error('[Scheduler] Unhandled error during collection:', err.message);
|
||||
lastRunResult = { success: false, error: err.message, run_count: runCount };
|
||||
return lastRunResult;
|
||||
} finally {
|
||||
isRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the scheduler (cron job + immediate first run).
|
||||
*/
|
||||
function startScheduler() {
|
||||
console.log(`[Scheduler] Starting proxy data collector`);
|
||||
const modeLabel = COLLECT_MODE === 'agent'
|
||||
? `SPECIFIC AGENT (${SPECIFIC_AGENT})`
|
||||
: COLLECT_MODE === 'agents'
|
||||
? `SPECIFIC AGENTS (${SPECIFIC_AGENTS.join(', ')})`
|
||||
: 'ALL AGENTS';
|
||||
console.log(`[Scheduler] Mode : ${modeLabel}`);
|
||||
console.log(`[Scheduler] Schedule : ${CRON_SCHEDULE} (every 5 minutes by default)`);
|
||||
|
||||
// Validate cron expression
|
||||
if (!cron.validate(CRON_SCHEDULE)) {
|
||||
console.error(`[Scheduler] Invalid cron expression: "${CRON_SCHEDULE}". Using default.`);
|
||||
}
|
||||
|
||||
// Start recurring cron job
|
||||
cron.schedule(CRON_SCHEDULE, () => {
|
||||
runCollection().catch(err => console.error('[Scheduler] Cron error:', err.message));
|
||||
});
|
||||
|
||||
console.log('[Scheduler] Cron job registered. Starting initial collection...');
|
||||
|
||||
// Initial run immediately on startup (async, do not block server start)
|
||||
setTimeout(async () => {
|
||||
// 1. Sync dictionary first
|
||||
// await netifyClient.syncApplicationDictionary();
|
||||
|
||||
// 2. Start normal telemetry collection
|
||||
runCollection().catch(err => console.error('[Scheduler] Initial run error:', err.message));
|
||||
}, 2000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current scheduler status (for REST API endpoint).
|
||||
*/
|
||||
function getStatus() {
|
||||
return {
|
||||
is_running: isRunning,
|
||||
run_count: runCount,
|
||||
last_run_at: lastRunAt?.toISOString() ?? null,
|
||||
collect_mode: COLLECT_MODE,
|
||||
agent_uuid: SPECIFIC_AGENT,
|
||||
agent_uuids: COLLECT_MODE === 'agents' ? SPECIFIC_AGENTS : [],
|
||||
agent_delay_ms: AGENT_DELAY_MS,
|
||||
cron_schedule: CRON_SCHEDULE,
|
||||
last_result: lastRunResult,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { startScheduler, runCollection, getStatus };
|
||||
// proxy/scheduler.js
|
||||
// Cron scheduler for automatic data collection from DPI API
|
||||
// Runs every 5 minutes, collecting data for all agents or a specific agent.
|
||||
|
||||
const cron = require('node-cron');
|
||||
const { collectAllAgents, collectSpecificAgent, collectSpecificAgents } = require('./collector');
|
||||
const { syncApplicationDictionary } = require('./netifyClient');
|
||||
|
||||
// Mode: 'all' = collect all agents, 'agent' = collect one specific agent, 'agents' = collect multiple agents
|
||||
const COLLECT_MODE = process.env.PROXY_COLLECT_MODE || 'all';
|
||||
const SPECIFIC_AGENT = process.env.PROXY_AGENT_UUID || null;
|
||||
const SPECIFIC_AGENTS = (process.env.PROXY_AGENT_UUIDS || '').split(',').map(s => s.trim()).filter(Boolean);
|
||||
const AGENT_DELAY_MS = parseInt(process.env.PROXY_AGENT_DELAY_MS || '5000');
|
||||
const CRON_SCHEDULE = process.env.PROXY_CRON_SCHEDULE || '*/5 * * * *';
|
||||
|
||||
// Capacity logging is an expensive full-scan aggregation. Run it at most once per
|
||||
// interval (default 24h) instead of every collection cycle to reduce CPU/DB load.
|
||||
const CAPACITY_LOG_INTERVAL_MS = parseInt(process.env.PROXY_CAPACITY_LOG_INTERVAL_MS || String(24 * 60 * 60 * 1000));
|
||||
|
||||
let isRunning = false;
|
||||
let lastRunAt = null;
|
||||
let lastRunResult = null;
|
||||
let runCount = 0;
|
||||
let lastCapacityLogAt = 0;
|
||||
|
||||
/**
|
||||
* Execute one collection cycle (called by cron and manual trigger).
|
||||
* Prevents concurrent runs with isRunning guard.
|
||||
*/
|
||||
async function runCollection() {
|
||||
if (isRunning) {
|
||||
console.log('[Scheduler] Skipping - previous run still in progress');
|
||||
return { skipped: true, reason: 'already_running' };
|
||||
}
|
||||
|
||||
isRunning = true;
|
||||
lastRunAt = new Date();
|
||||
runCount++;
|
||||
|
||||
try {
|
||||
let result;
|
||||
if (COLLECT_MODE === 'agent' && SPECIFIC_AGENT) {
|
||||
console.log(`[Scheduler] Run #${runCount} - Mode: SPECIFIC AGENT (${SPECIFIC_AGENT})`);
|
||||
result = await collectSpecificAgent(SPECIFIC_AGENT);
|
||||
} else if (COLLECT_MODE === 'agents' && SPECIFIC_AGENTS.length > 0) {
|
||||
console.log(`[Scheduler] Run #${runCount} - Mode: SPECIFIC AGENTS (${SPECIFIC_AGENTS.join(', ')})`);
|
||||
result = await collectSpecificAgents(SPECIFIC_AGENTS, AGENT_DELAY_MS);
|
||||
} else {
|
||||
console.log(`[Scheduler] Run #${runCount} - Mode: ALL AGENTS`);
|
||||
result = await collectAllAgents();
|
||||
}
|
||||
lastRunResult = { ...result, run_count: runCount };
|
||||
|
||||
// Log MongoDB database capacity usage (expensive full-scan aggregation).
|
||||
// Only run periodically (default: every 24h) to avoid high CPU/DB load each cycle.
|
||||
const now = Date.now();
|
||||
if (now - lastCapacityLogAt >= CAPACITY_LOG_INTERVAL_MS) {
|
||||
lastCapacityLogAt = now;
|
||||
const { logCapacityStats } = require('./db/capacityTracker');
|
||||
await logCapacityStats(`[PROXY] [MongoDB] Capacity Used after Run #${runCount}:`);
|
||||
}
|
||||
|
||||
return lastRunResult;
|
||||
} catch (err) {
|
||||
console.error('[Scheduler] Unhandled error during collection:', err.message);
|
||||
lastRunResult = { success: false, error: err.message, run_count: runCount };
|
||||
return lastRunResult;
|
||||
} finally {
|
||||
isRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the scheduler (cron job + immediate first run).
|
||||
*/
|
||||
function startScheduler() {
|
||||
console.log(`[Scheduler] Starting proxy data collector`);
|
||||
const modeLabel = COLLECT_MODE === 'agent'
|
||||
? `SPECIFIC AGENT (${SPECIFIC_AGENT})`
|
||||
: COLLECT_MODE === 'agents'
|
||||
? `SPECIFIC AGENTS (${SPECIFIC_AGENTS.join(', ')})`
|
||||
: 'ALL AGENTS';
|
||||
console.log(`[Scheduler] Mode : ${modeLabel}`);
|
||||
console.log(`[Scheduler] Schedule : ${CRON_SCHEDULE} (every 5 minutes by default)`);
|
||||
|
||||
// Validate cron expression
|
||||
if (!cron.validate(CRON_SCHEDULE)) {
|
||||
console.error(`[Scheduler] Invalid cron expression: "${CRON_SCHEDULE}". Using default.`);
|
||||
}
|
||||
|
||||
// Start recurring cron job
|
||||
cron.schedule(CRON_SCHEDULE, () => {
|
||||
runCollection().catch(err => console.error('[Scheduler] Cron error:', err.message));
|
||||
});
|
||||
|
||||
console.log('[Scheduler] Cron job registered. Starting initial collection...');
|
||||
|
||||
// Initial run immediately on startup (async, do not block server start)
|
||||
setTimeout(async () => {
|
||||
// 1. Sync dictionary first
|
||||
try {
|
||||
await syncApplicationDictionary();
|
||||
} catch (err) {
|
||||
console.error('[Scheduler] Error syncing application dictionary:', err.message);
|
||||
}
|
||||
|
||||
// 2. Start normal telemetry collection
|
||||
runCollection().catch(err => console.error('[Scheduler] Initial run error:', err.message));
|
||||
}, 2000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current scheduler status (for REST API endpoint).
|
||||
*/
|
||||
function getStatus() {
|
||||
return {
|
||||
is_running: isRunning,
|
||||
run_count: runCount,
|
||||
last_run_at: lastRunAt?.toISOString() ?? null,
|
||||
collect_mode: COLLECT_MODE,
|
||||
agent_uuid: SPECIFIC_AGENT,
|
||||
agent_uuids: COLLECT_MODE === 'agents' ? SPECIFIC_AGENTS : [],
|
||||
agent_delay_ms: AGENT_DELAY_MS,
|
||||
cron_schedule: CRON_SCHEDULE,
|
||||
last_result: lastRunResult,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { startScheduler, runCollection, getStatus };
|
||||
@@ -0,0 +1,71 @@
|
||||
// test_api.js - Tests the actual metadata-detail API endpoint
|
||||
// Run: node proxy/test_api.js
|
||||
const http = require('http');
|
||||
|
||||
function request(path) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const options = {
|
||||
hostname: 'localhost',
|
||||
port: 3001,
|
||||
path,
|
||||
method: 'GET',
|
||||
};
|
||||
const req = http.request(options, res => {
|
||||
let body = '';
|
||||
res.on('data', chunk => body += chunk);
|
||||
res.on('end', () => {
|
||||
try { resolve({ status: res.statusCode, data: JSON.parse(body) }); }
|
||||
catch (e) { resolve({ status: res.statusCode, raw: body }); }
|
||||
});
|
||||
});
|
||||
req.on('error', reject);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// Test 1: netbios_hostname for 10.6.10.44
|
||||
console.log('\n=== TEST 1: netbios_hostname=10.6.10.44 ===');
|
||||
try {
|
||||
const r1 = await request('/api/dashboard/metadata-detail?type=netbios_hostname&value=10.6.10.44');
|
||||
console.log('Status:', r1.status);
|
||||
if (r1.data) {
|
||||
console.log('Count:', r1.data.count);
|
||||
console.log('First 3 rows:', JSON.stringify(r1.data.data?.slice(0, 3), null, 2));
|
||||
} else {
|
||||
console.log('Raw:', r1.raw?.slice(0, 500));
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('ERROR (maybe backend is on different port):', e.message);
|
||||
}
|
||||
|
||||
// Test 2: Try port 3000 (Next.js API routes)
|
||||
console.log('\n=== TEST 2: via Next.js port 3000 ===');
|
||||
try {
|
||||
const r2 = await request('/api/dashboard/metadata-detail?type=netbios_hostname&value=10.6.10.44');
|
||||
const options2 = { hostname: 'localhost', port: 3000, path: '/api/dashboard/metadata-detail?type=netbios_hostname&value=10.6.10.44', method: 'GET' };
|
||||
const r3 = await new Promise((resolve, reject) => {
|
||||
const req = http.request(options2, res => {
|
||||
let body = '';
|
||||
res.on('data', chunk => body += chunk);
|
||||
res.on('end', () => {
|
||||
try { resolve({ status: res.statusCode, data: JSON.parse(body) }); }
|
||||
catch (e) { resolve({ status: res.statusCode, raw: body?.slice(0, 500) }); }
|
||||
});
|
||||
});
|
||||
req.on('error', reject);
|
||||
req.end();
|
||||
});
|
||||
console.log('Port 3000 - Status:', r3.status);
|
||||
if (r3.data) {
|
||||
console.log('Count:', r3.data.count);
|
||||
console.log('First 3 rows:', JSON.stringify(r3.data.data?.slice(0, 3), null, 2));
|
||||
} else {
|
||||
console.log('Raw:', r3.raw);
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('Port 3000 ERROR:', e.message);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
@@ -0,0 +1 @@
|
||||
const mongoose = require('mongoose'); require('dotenv').config({ path: '../.env.local' }); const { LookupApp } = require('./models/Schemas'); async function test() { await mongoose.connect(process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017/backone_dpi'); const sample = await LookupApp.findOne({ tag: /youtube/i }).lean(); console.log(JSON.stringify(sample, null, 2)); await mongoose.disconnect(); } test().catch(console.error);
|
||||
@@ -0,0 +1,89 @@
|
||||
// test_metadata_detail.js - Test after restart
|
||||
// Run: node proxy/test_metadata_detail.js
|
||||
const http = require('http');
|
||||
|
||||
function post(path, body) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const data = JSON.stringify(body);
|
||||
const options = {
|
||||
hostname: 'localhost', port: 3001, path, method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data) },
|
||||
};
|
||||
const req = http.request(options, res => {
|
||||
let buf = '';
|
||||
res.on('data', c => buf += c);
|
||||
res.on('end', () => {
|
||||
try { resolve({ status: res.statusCode, headers: res.headers, data: JSON.parse(buf) }); }
|
||||
catch { resolve({ status: res.statusCode, raw: buf }); }
|
||||
});
|
||||
});
|
||||
req.on('error', reject);
|
||||
req.write(data);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
function get(path, cookie) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const options = {
|
||||
hostname: 'localhost', port: 3001, path, method: 'GET',
|
||||
headers: cookie ? { Cookie: cookie } : {},
|
||||
};
|
||||
const req = http.request(options, res => {
|
||||
let buf = '';
|
||||
res.on('data', c => buf += c);
|
||||
res.on('end', () => {
|
||||
try { resolve({ status: res.statusCode, data: JSON.parse(buf) }); }
|
||||
catch { resolve({ status: res.statusCode, raw: buf?.slice(0, 300) }); }
|
||||
});
|
||||
});
|
||||
req.on('error', reject);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// Step 1: Login to get cookie
|
||||
console.log('=== Step 1: Login ===');
|
||||
const login = await post('/api/auth/login', { username: 'admin', password: 'admin123' });
|
||||
console.log('Login status:', login.status);
|
||||
|
||||
const setCookie = login.headers?.['set-cookie'];
|
||||
let cookie = '';
|
||||
if (setCookie) {
|
||||
cookie = setCookie.map(c => c.split(';')[0]).join('; ');
|
||||
console.log('Cookie obtained:', cookie.slice(0, 60) + '...');
|
||||
} else {
|
||||
console.log('No cookie received. Auth response:', JSON.stringify(login.data));
|
||||
// Try with a known admin credential
|
||||
}
|
||||
|
||||
// Step 2: Test health
|
||||
console.log('\n=== Step 2: Health Check ===');
|
||||
const health = await get('/api/health', cookie);
|
||||
console.log('Health:', health.status, JSON.stringify(health.data));
|
||||
|
||||
// Step 3: Test metadata-detail netbios_hostname
|
||||
console.log('\n=== Step 3: metadata-detail netbios_hostname=10.6.10.44 ===');
|
||||
const r = await get('/api/dashboard/metadata-detail?type=netbios_hostname&value=10.6.10.44', cookie);
|
||||
console.log('Status:', r.status);
|
||||
if (r.data) {
|
||||
console.log('Count (should be 1):', r.data.count);
|
||||
console.log('Data:', JSON.stringify(r.data.data, null, 2));
|
||||
} else {
|
||||
console.log('Raw:', r.raw);
|
||||
}
|
||||
|
||||
// Step 4: Verify DB has 1 doc for 10.6.10.44
|
||||
console.log('\n=== Step 4: Direct DB verification ===');
|
||||
const mongoose = require('mongoose');
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
|
||||
await mongoose.connect(process.env.MONGODB_URI || 'mongodb://localhost:27017/backone');
|
||||
const db = mongoose.connection.db;
|
||||
const cnt = await db.collection('devicestats').countDocuments({ ip_address: '10.6.10.44' });
|
||||
console.log('DB count for 10.6.10.44:', cnt, '(expected: 1)');
|
||||
await mongoose.disconnect();
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
@@ -0,0 +1 @@
|
||||
const mongoose = require('mongoose'); require('dotenv').config({ path: '../.env.local' }); const { syncApplicationDictionary } = require('./netifyClient'); const { LookupApp } = require('./models/Schemas'); async function test() { await mongoose.connect(process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017/backone_dpi'); console.log('Connected to DB'); await syncApplicationDictionary(); const count = await LookupApp.countDocuments(); console.log('Total LookupApps in DB:', count); await mongoose.disconnect(); } test().catch(console.error);
|
||||
@@ -0,0 +1,56 @@
|
||||
// verify_fix.js - Directly verify the MongoDB aggregation returns correct results
|
||||
// This simulates what the backend metadata-detail endpoint does after the fix.
|
||||
// Run: node proxy/verify_fix.js
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.join(__dirname, '..', '.env.local') });
|
||||
const mongoose = require('mongoose');
|
||||
|
||||
async function run() {
|
||||
await mongoose.connect(process.env.MONGODB_URI || 'mongodb://localhost:27017/backone');
|
||||
const db = mongoose.connection.db;
|
||||
const col = db.collection('devicestats');
|
||||
|
||||
const testValues = ['10.6.10.44'];
|
||||
// Also find other common device_labels to test
|
||||
const sample = await col.find({}).limit(20).toArray();
|
||||
const labels = [...new Set(sample.map(d => d.device_label).filter(Boolean))];
|
||||
console.log('Sample device_labels to test:', labels.slice(0, 5));
|
||||
|
||||
for (const value of [...testValues, ...labels.slice(0, 3)]) {
|
||||
// Count raw docs matching
|
||||
const rawCount = await col.countDocuments({ device_label: value });
|
||||
|
||||
// Simulate the new aggregation pipeline
|
||||
const aggResult = await col.aggregate([
|
||||
{ $match: { device_label: value } },
|
||||
{ $sort: { timestamp: -1 } },
|
||||
{ $group: {
|
||||
_id: '$ip_address',
|
||||
mac_address: { $first: '$mac_address' },
|
||||
device_label: { $first: '$device_label' },
|
||||
download: { $max: '$download' },
|
||||
upload: { $max: '$upload' },
|
||||
}},
|
||||
{ $sort: { download: -1 } },
|
||||
]).toArray();
|
||||
|
||||
const status = rawCount > aggResult.length ? '✅ FIXED (was duplicated)' : '✓ OK';
|
||||
console.log(`\ndevice_label="${value}": raw=${rawCount} rows → aggregated=${aggResult.length} unique devices ${status}`);
|
||||
if (aggResult.length > 0) {
|
||||
console.log(' First result:', JSON.stringify(aggResult[0], null, 2));
|
||||
}
|
||||
}
|
||||
|
||||
// Verify unique index exists
|
||||
const indexes = await col.indexes();
|
||||
const uniqueIdx = indexes.find(i => i.unique && i.key.agent_uuid && i.key.ip_address);
|
||||
console.log('\n=== Unique Index on (agent_uuid, ip_address):', uniqueIdx ? `✅ EXISTS (${uniqueIdx.name})` : '❌ MISSING');
|
||||
|
||||
// Final count
|
||||
const total = await col.countDocuments();
|
||||
console.log('=== Total DeviceStat docs:', total);
|
||||
|
||||
await mongoose.disconnect();
|
||||
}
|
||||
|
||||
run().catch(err => { console.error(err.message); process.exit(1); });
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 246 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 246 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 246 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 246 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 246 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 246 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 429 KiB |
+21
-21
@@ -1,21 +1,21 @@
|
||||
<svg width="200" height="200" viewBox="0 0 200 200" xmlns="http://www.w3.org/2000/svg">
|
||||
<defs>
|
||||
<linearGradient id="bgGradient" x1="0%" y1="0%" x2="100%" y2="100%">
|
||||
<stop offset="0%" stop-color="#f00a38" />
|
||||
<stop offset="50%" stop-color="#7a0799" />
|
||||
<stop offset="100%" stop-color="#0400ff" />
|
||||
</linearGradient>
|
||||
</defs>
|
||||
|
||||
<!-- Gradient Circle Background -->
|
||||
<circle cx="100" cy="100" r="100" fill="url(#bgGradient)" />
|
||||
|
||||
<!-- White Abstract Shape (BackOne Logo) -->
|
||||
<path d="M 50 45
|
||||
L 90 45
|
||||
C 145 45, 165 65, 165 105
|
||||
C 165 155, 110 180, 55 180
|
||||
C 115 160, 125 100, 50 95
|
||||
Z"
|
||||
fill="#FFFFFF" />
|
||||
</svg>
|
||||
<svg width="200" height="200" viewBox="0 0 200 200" xmlns="http://www.w3.org/2000/svg">
|
||||
<defs>
|
||||
<linearGradient id="bgGradient" x1="0%" y1="0%" x2="100%" y2="100%">
|
||||
<stop offset="0%" stop-color="#f00a38" />
|
||||
<stop offset="50%" stop-color="#7a0799" />
|
||||
<stop offset="100%" stop-color="#0400ff" />
|
||||
</linearGradient>
|
||||
</defs>
|
||||
|
||||
<!-- Gradient Circle Background -->
|
||||
<circle cx="100" cy="100" r="100" fill="url(#bgGradient)" />
|
||||
|
||||
<!-- White Abstract Shape (BackOne Logo) -->
|
||||
<path d="M 50 45
|
||||
L 90 45
|
||||
C 145 45, 165 65, 165 105
|
||||
C 165 155, 110 180, 55 180
|
||||
C 115 160, 125 100, 50 95
|
||||
Z"
|
||||
fill="#FFFFFF" />
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 701 B After Width: | Height: | Size: 722 B |
Loaded 100 of 385 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user