feat: complete deployment codebase for BackOne DPI dashboard with dynamic telemetry, rebranding, database size tracking, and modular design
This commit is contained in:
Rafif-Riqullah-Siregar committed
1 parent
6cf3c6c7e5
commit
c316f3171b
226 files changed
+21152
-8397
No files matched your search
+223
-65
@@ -1,97 +1,255 @@
|
||||
// backend/models/Schemas.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// MongoDB Schemas untuk BackOne Backend (READ-ONLY)
|
||||
//
|
||||
// PENTING: Schema ini harus sinkron dengan proxy/models/Schemas.js
|
||||
// Proxy yang MENULIS data, backend yang MEMBACA data.
|
||||
//
|
||||
// Setiap dokumen di-tag dengan:
|
||||
// agent_uuid → identifikasi Network Agent spesifik (isolasi per tenant)
|
||||
// site_uuid → identifikasi site DPI (BackOne)
|
||||
// timestamp → waktu data dikumpulkan
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const mongoose = require('mongoose');
|
||||
|
||||
const baseOptions = { timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' } };
|
||||
const baseOptions = {
|
||||
timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
|
||||
};
|
||||
|
||||
// Schema for Timeline / General Summary
|
||||
// ─── Bandwidth Summary (per agent, per collection cycle) ───────────────────────
|
||||
const SummarySchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
total_devices: Number,
|
||||
active_flows: Number,
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true }, // null = global/all agents
|
||||
site_uuid: { type: String, index: true },
|
||||
bandwidth_down: Number,
|
||||
bandwidth_up: Number,
|
||||
bandwidth_up: Number,
|
||||
active_flows: Number,
|
||||
download_speed: Number,
|
||||
upload_speed: Number,
|
||||
total_threats: Number
|
||||
upload_speed: Number,
|
||||
total_devices: Number,
|
||||
total_threats: Number,
|
||||
packet_drops: Number,
|
||||
peak_flow_rate: Number,
|
||||
cpu_usage: Number,
|
||||
memory_usage: Number,
|
||||
queue_depth: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// Schema for Top Apps
|
||||
// ─── Top Applications (per agent) ─────────────────────────────────────────────
|
||||
const AppStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true },
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
app_label: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number
|
||||
site_uuid: { type: String, index: true },
|
||||
app_label: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// Schema for Protocols
|
||||
// ─── Protocol Statistics (per agent) ──────────────────────────────────────────
|
||||
const ProtocolStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
protocol_name: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
protocol_label: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// Schema for Devices
|
||||
// ─── Discovered Devices (per agent, includes IP + MAC + device info) ───────────
|
||||
const DeviceStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
ip_address: { type: String, required: true, index: true },
|
||||
mac_address: { type: String, index: true },
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
ip_address: { type: String, required: true, index: true },
|
||||
mac_address: { type: String, index: true },
|
||||
device_label: String,
|
||||
device_type: String,
|
||||
os_label: String,
|
||||
device_type: String,
|
||||
os_label: String,
|
||||
manufacturer: String,
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
last_seen: String
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
last_seen: String,
|
||||
}, baseOptions);
|
||||
|
||||
// Schema for Flows
|
||||
// ─── Network Flows (per agent) ─────────────────────────────────────────────────
|
||||
const FlowSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true },
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
flow_id: String,
|
||||
src_ip: { type: String, index: true },
|
||||
src_mac: String,
|
||||
dst_ip: { type: String, index: true },
|
||||
dst_port: Number,
|
||||
protocol: String,
|
||||
app_label: String,
|
||||
domain: String,
|
||||
download: Number,
|
||||
upload: Number,
|
||||
site_uuid: { type: String, index: true },
|
||||
flow_id: String,
|
||||
src_ip: { type: String, index: true },
|
||||
src_mac: String,
|
||||
dst_ip: { type: String, index: true },
|
||||
dst_port: Number,
|
||||
protocol: String,
|
||||
app_label: String,
|
||||
domain: String,
|
||||
download: Number,
|
||||
upload: Number,
|
||||
first_seen: String,
|
||||
last_seen: String
|
||||
last_seen: String,
|
||||
}, baseOptions);
|
||||
|
||||
// Schema for Threats
|
||||
// ─── Cyber Threats (per agent) ─────────────────────────────────────────────────
|
||||
const ThreatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
threat_id: String,
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
threat_type: String,
|
||||
severity: String,
|
||||
src_ip: String,
|
||||
dst_ip: String,
|
||||
details: mongoose.Schema.Types.Mixed
|
||||
severity: String,
|
||||
src_ip: String,
|
||||
dst_ip: String,
|
||||
dst_port: Number,
|
||||
protocol: String,
|
||||
description: String,
|
||||
event_at: String,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── App Categories (per agent) ───────────────────────────────────────────────
|
||||
const AppCategoryStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
category_label: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── System Events (per agent) ─────────────────────────────────────────────────
|
||||
const EventSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
event_id: Number,
|
||||
event_type: String,
|
||||
severity: String,
|
||||
description: String,
|
||||
category_label: String,
|
||||
ip_address: String,
|
||||
mac_address: String,
|
||||
event_at: Date,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Compound Indexes for common dashboard queries ─────────────────────────────
|
||||
// ─── TLS Versions (per agent) ──────────────────────────────────────────────────
|
||||
const TlsVersionStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
tls_version: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── TLS Ciphers (per agent) ───────────────────────────────────────────────────
|
||||
const TlsCipherStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
tls_cipher: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── TLS Security (per agent) ──────────────────────────────────────────────────
|
||||
const TlsSecurityStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
tls_security: { type: String, required: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Country Traffic Stats (per agent) ────────────────────────────────────────
|
||||
const CountryStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
country_code: { type: String, required: true },
|
||||
country_name: { type: String, default: '' },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
|
||||
// ─── Compound Indexes for common dashboard queries ─────────────────────────────
|
||||
SummarySchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
AppStatSchema.index({ agent_uuid: 1, timestamp: -1, download: -1 });
|
||||
DeviceStatSchema.index({ agent_uuid: 1, ip_address: 1 }, { unique: true });
|
||||
FlowSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
FlowSchema.index({ agent_uuid: 1, flow_id: 1 });
|
||||
FlowSchema.index({ agent_uuid: 1, protocol: 1, timestamp: -1 });
|
||||
FlowSchema.index({ agent_uuid: 1, domain: 1, timestamp: -1 });
|
||||
ThreatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
AppCategoryStatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
EventSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
TlsVersionStatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
TlsCipherStatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
TlsSecurityStatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
CountryStatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
|
||||
const CustomDeviceLabelSchema = new mongoose.Schema({
|
||||
mac_address: { type: String, required: true, unique: true, index: true },
|
||||
device_label: { type: String, required: true },
|
||||
}, baseOptions);
|
||||
|
||||
// ── Per-Device Per-Application Stats (synced from proxy) ─────────────────
|
||||
const DeviceAppStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
ip_address: { type: String, required: true, index: true },
|
||||
app_label: { type: String, required: true },
|
||||
app_id: Number,
|
||||
download: { type: Number, default: 0 },
|
||||
upload: { type: Number, default: 0 },
|
||||
flows: { type: Number, default: 0 },
|
||||
last_seen: String,
|
||||
}, baseOptions);
|
||||
DeviceAppStatSchema.index({ agent_uuid: 1, ip_address: 1, timestamp: -1 });
|
||||
DeviceAppStatSchema.index({ ip_address: 1, app_label: 1, timestamp: -1 });
|
||||
|
||||
// ─── View As Audit Logs ────────────────────────────────────────────────────────
|
||||
const ViewAsLogSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, default: Date.now, index: true },
|
||||
admin_id: { type: String, required: true },
|
||||
admin_username: { type: String, required: true },
|
||||
admin_role: String,
|
||||
agent_uuid: { type: String, required: true },
|
||||
agent_label: String,
|
||||
end_timestamp: Date,
|
||||
duration: Number, // duration in seconds
|
||||
}, baseOptions);
|
||||
|
||||
|
||||
// ─── DPI Telemetry Property Schemas (SNI, SSL, QUIC, SSH, mDNS, DHCP, UA, BT)
|
||||
// Split into SchemasTelemetry.js to keep this file under 256 lines.
|
||||
const telemetrySchemas = require('./SchemasTelemetry');
|
||||
|
||||
module.exports = {
|
||||
Summary: mongoose.model('Summary', SummarySchema),
|
||||
AppStat: mongoose.model('AppStat', AppStatSchema),
|
||||
Summary: mongoose.model('Summary', SummarySchema),
|
||||
AppStat: mongoose.model('AppStat', AppStatSchema),
|
||||
ProtocolStat: mongoose.model('ProtocolStat', ProtocolStatSchema),
|
||||
DeviceStat: mongoose.model('DeviceStat', DeviceStatSchema),
|
||||
Flow: mongoose.model('Flow', FlowSchema),
|
||||
Threat: mongoose.model('Threat', ThreatSchema)
|
||||
DeviceStat: mongoose.model('DeviceStat', DeviceStatSchema),
|
||||
DeviceAppStat: mongoose.model('DeviceAppStat', DeviceAppStatSchema),
|
||||
Flow: mongoose.model('Flow', FlowSchema),
|
||||
Threat: mongoose.model('Threat', ThreatSchema),
|
||||
CustomDeviceLabel: mongoose.model('CustomDeviceLabel', CustomDeviceLabelSchema),
|
||||
AppCategoryStat: mongoose.model('AppCategoryStat', AppCategoryStatSchema),
|
||||
Event: mongoose.model('Event', EventSchema),
|
||||
TlsVersionStat: mongoose.model('TlsVersionStat', TlsVersionStatSchema),
|
||||
TlsCipherStat: mongoose.model('TlsCipherStat', TlsCipherStatSchema),
|
||||
TlsSecurityStat: mongoose.model('TlsSecurityStat', TlsSecurityStatSchema),
|
||||
CountryStat: mongoose.model('CountryStat', CountryStatSchema),
|
||||
ViewAsLog: mongoose.model('ViewAsLog', ViewAsLogSchema),
|
||||
...telemetrySchemas,
|
||||
};
|
||||
@@ -0,0 +1,81 @@
|
||||
// backend/models/SchemasTelemetry.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// DPI Telemetry Property Schemas for BackOne Backend (READ-ONLY).
|
||||
// Split from Schemas.js to keep files under 256 lines.
|
||||
// Must stay in sync with proxy/models/SchemasTelemetry.js.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const mongoose = require('mongoose');
|
||||
|
||||
const baseOptions = {
|
||||
timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
|
||||
};
|
||||
|
||||
// ─── Helper: build a consistent DPI property schema ───────────────────────────
|
||||
function dpiPropertySchema(fieldName) {
|
||||
const fields = {
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
};
|
||||
fields[fieldName] = { type: String, required: true };
|
||||
const schema = new mongoose.Schema(fields, baseOptions);
|
||||
schema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
return schema;
|
||||
}
|
||||
|
||||
// ─── DHCP Fingerprints (dhcp_class) ──────────────────────────────────────────
|
||||
const DhcpFingerprintStatSchema = dpiPropertySchema('fingerprint');
|
||||
|
||||
// ─── HTTP User Agents (http_useragent) ────────────────────────────────────────
|
||||
const HttpUserAgentStatSchema = dpiPropertySchema('user_agent');
|
||||
|
||||
// ─── BitTorrent Info Hashes (bittorrent_info_hash) ────────────────────────────
|
||||
const BittorrentHashStatSchema = new mongoose.Schema({
|
||||
timestamp: { type: Date, required: true, index: true, expires: '7d' },
|
||||
agent_uuid: { type: String, index: true },
|
||||
site_uuid: { type: String, index: true },
|
||||
info_hash: { type: String, required: true },
|
||||
label: { type: String },
|
||||
download: Number,
|
||||
upload: Number,
|
||||
flows: Number,
|
||||
}, baseOptions);
|
||||
BittorrentHashStatSchema.index({ agent_uuid: 1, timestamp: -1 });
|
||||
|
||||
// ─── HTTPS SNI Hostnames (https_sni_hostname) ─────────────────────────────────
|
||||
const SniHostnameStatSchema = dpiPropertySchema('sni_hostname');
|
||||
|
||||
// ─── SSL Server Common Names (ssl_server_cn) ──────────────────────────────────
|
||||
const SslServerCnStatSchema = dpiPropertySchema('ssl_server_cn');
|
||||
|
||||
// ─── QUIC Hostnames (quic_hostname) ───────────────────────────────────────────
|
||||
const QuicHostnameStatSchema = dpiPropertySchema('quic_hostname');
|
||||
|
||||
// ─── SSH Clients (ssh_client) ─────────────────────────────────────────────────
|
||||
const SshClientStatSchema = dpiPropertySchema('ssh_client');
|
||||
|
||||
// ─── SSH Servers (ssh_server) ─────────────────────────────────────────────────
|
||||
const SshServerStatSchema = dpiPropertySchema('ssh_server');
|
||||
|
||||
// ─── mDNS Hostnames (mdns_hostname) ───────────────────────────────────────────
|
||||
const MdnsHostnameStatSchema = dpiPropertySchema('mdns_hostname');
|
||||
|
||||
// ─── SSL Subject Alternative Names (ssl_subject_alt_name) ──────────────────────
|
||||
const SslSubjectAltNameStatSchema = dpiPropertySchema('alt_name');
|
||||
|
||||
module.exports = {
|
||||
DhcpFingerprintStat: mongoose.model('DhcpFingerprintStat', DhcpFingerprintStatSchema),
|
||||
HttpUserAgentStat: mongoose.model('HttpUserAgentStat', HttpUserAgentStatSchema),
|
||||
BittorrentHashStat: mongoose.model('BittorrentHashStat', BittorrentHashStatSchema),
|
||||
SniHostnameStat: mongoose.model('SniHostnameStat', SniHostnameStatSchema),
|
||||
SslServerCnStat: mongoose.model('SslServerCnStat', SslServerCnStatSchema),
|
||||
QuicHostnameStat: mongoose.model('QuicHostnameStat', QuicHostnameStatSchema),
|
||||
SshClientStat: mongoose.model('SshClientStat', SshClientStatSchema),
|
||||
SshServerStat: mongoose.model('SshServerStat', SshServerStatSchema),
|
||||
MdnsHostnameStat: mongoose.model('MdnsHostnameStat', MdnsHostnameStatSchema),
|
||||
SslSubjectAltNameStat: mongoose.model('SslSubjectAltNameStat', SslSubjectAltNameStatSchema),
|
||||
};
|
||||
@@ -0,0 +1,42 @@
|
||||
// backend/models/User.js
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// MongoDB User Schema untuk BackOne Authentication
|
||||
//
|
||||
// Roles:
|
||||
// SUPER_ADMIN → akses semua data semua agent
|
||||
// AGENT_VIEWER → akses data agent_uuid tertentu saja (multi-tenant isolation)
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const mongoose = require('mongoose');
|
||||
|
||||
const UserSchema = new mongoose.Schema({
|
||||
username: { type: String, required: true, unique: true, trim: true },
|
||||
password_hash: { type: String, required: true },
|
||||
account_name: { type: String, default: null },
|
||||
profile_picture: { type: String, default: null },
|
||||
role: { type: String, enum: ['SUPER_ADMIN', 'TENANT_ADMIN', 'SOC_ANALYST', 'ENGINEER', 'AGENT_VIEWER'], default: 'AGENT_VIEWER' },
|
||||
site_uuid: { type: String, default: null, index: true },
|
||||
agent_uuid: { type: String, default: null },
|
||||
is_active: { type: Boolean, default: true },
|
||||
}, {
|
||||
timestamps: { createdAt: 'created_at', updatedAt: 'updated_at' }
|
||||
});
|
||||
|
||||
// Virtual 'id' getter (returns string version of _id for backward compat)
|
||||
UserSchema.virtual('id').get(function () {
|
||||
return this._id.toString();
|
||||
});
|
||||
|
||||
UserSchema.set('toJSON', {
|
||||
virtuals: true,
|
||||
transform: (doc, ret) => {
|
||||
delete ret.__v;
|
||||
delete ret.password_hash; // Never leak password hash
|
||||
return ret;
|
||||
}
|
||||
});
|
||||
|
||||
// Compound index for agent_uuid lookup
|
||||
UserSchema.index({ agent_uuid: 1, is_active: 1 });
|
||||
|
||||
module.exports = mongoose.model('User', UserSchema);
|
||||
Reference in new issue
Block a user