From 4e8ae537e80d91a1d10becbc3de0439d0774de2c Mon Sep 17 00:00:00 2001 From: dsutanto Date: Mon, 20 Jul 2026 13:44:37 +0700 Subject: [PATCH 1/4] Update Backend query limit to 10000 for Application lookup --- backend/routes/appDetailsDpiHelper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/routes/appDetailsDpiHelper.js b/backend/routes/appDetailsDpiHelper.js index ade6701..7a76ab1 100644 --- a/backend/routes/appDetailsDpiHelper.js +++ b/backend/routes/appDetailsDpiHelper.js @@ -44,7 +44,7 @@ async function populateAppCache(BASE_URL, token, siteUuid) { const headers = { 'x-api-key': token, 'Accept': 'application/json' }; if (siteUuid) headers['x-net-site'] = siteUuid; const res = await axios.get(`${BASE_URL}/lookup/applications`, { - headers, params: { settings_limit: 5000 }, timeout: 20000 + headers, params: { settings_limit: 10000 }, timeout: 20000 }); appLookupCache = {}; if (res.data && Array.isArray(res.data.data)) { -- 2.54.0 From fa14860758a91f5216a9103fea3f9309d8134402 Mon Sep 17 00:00:00 2001 From: dsutanto Date: Mon, 20 Jul 2026 13:44:53 +0700 Subject: [PATCH 2/4] Update Dockerfile to be build from parent directory --- backend/Dockerfile.bun | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/backend/Dockerfile.bun b/backend/Dockerfile.bun index 55545b7..2a5595a 100644 --- a/backend/Dockerfile.bun +++ b/backend/Dockerfile.bun @@ -2,18 +2,13 @@ FROM oven/bun:1-alpine WORKDIR /app -# Install dependencies first (layer caching) -# bun install is compatible with npm package.json / package-lock.json -COPY package*.json ./ +COPY backend/package*.json ./ RUN bun install --production -# Copy application source -COPY . . +COPY backend/ . -# Expose backend API port EXPOSE 3001 -# Health check HEALTHCHECK --interval=30s --timeout=10s --start-period=20s --retries=3 \ CMD bun -e "require('http').get('http://localhost:3001/api/health', r => r.statusCode === 200 ? process.exit(0) : process.exit(1)).on('error', () => process.exit(1))" -- 2.54.0 From 7a87d4d48daf81910e79034323813362f911e56f Mon Sep 17 00:00:00 2001 From: dsutanto Date: Mon, 20 Jul 2026 13:45:08 +0700 Subject: [PATCH 3/4] Update Dockerfile to be build from parent directory --- proxy/Dockerfile.bun | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/proxy/Dockerfile.bun b/proxy/Dockerfile.bun index 80a56ad..2cdfd00 100644 --- a/proxy/Dockerfile.bun +++ b/proxy/Dockerfile.bun @@ -2,18 +2,13 @@ FROM oven/bun:1-alpine WORKDIR /app -# Install dependencies first (layer caching) -# bun install is compatible with npm package.json / package-lock.json -COPY package*.json ./ +COPY proxy/package*.json ./ RUN bun install --production -# Copy application source -COPY . . +COPY proxy/ . -# Expose proxy REST API port EXPOSE 4000 -# Health check HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \ CMD bun -e "require('http').get('http://localhost:4000/health', r => r.statusCode === 200 ? process.exit(0) : process.exit(1)).on('error', () => process.exit(1))" -- 2.54.0 From 31dc77fd67e4771d682a42cc3494353142b9289b Mon Sep 17 00:00:00 2001 From: dsutanto Date: Mon, 20 Jul 2026 13:45:22 +0700 Subject: [PATCH 4/4] Update collector to not use authentication --- proxy/collector.js | 43 ++++++++++++++++++++++++++++++++++++++++++- proxy/scheduler.js | 15 ++++++++++++--- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/proxy/collector.js b/proxy/collector.js index 5017b69..a31f802 100644 --- a/proxy/collector.js +++ b/proxy/collector.js @@ -18,6 +18,8 @@ const { } = require('./collectorHelperDpi2'); const { Summary, AppStat } = require('./models/Schemas'); +const { LookupApp } = require('./models/SchemasAux'); +const { BASE_URL } = require('./netifyClientCore'); const { pruneOldData } = require('./dataRetention'); const SITE_UUIDS_STR = process.env.NETIFY_SITE_UUIDS || process.env.NETIFY_SITE_UUID; @@ -255,8 +257,47 @@ async function collectSpecificAgents(agentUuids, siteUuid = SITE_UUIDS[0], delay return { success: true, mode: 'specific_agents', agents_count: agentUuids.length, successful, results }; } +async function populateLookupApps(siteUuid = '') { + try { + const axios = require('axios'); + const headers = { 'Accept': 'application/json' }; + const res = await axios.get(`${BASE_URL}/lookup/applications`, { + headers, params: { settings_limit: 10000 }, timeout: 20000 + }); + const apps = res.data?.data; + if (!apps || !Array.isArray(apps)) { + console.log('[LookupApp] No application data from DPI API'); + return { success: false, count: 0 }; + } + let upserted = 0; + for (const a of apps) { + if (!a.id) continue; + const doc = { + id: a.id, + tag: a.tag || '', + label: a.label || '', + name: a.full_label || a.label || '', + full_name: a.full_label || '', + description: a.description || '', + favicon: a.favicon || '', + icon: a.icon || '', + logo: a.logo || '', + application_category: a.category || null + }; + await LookupApp.updateOne({ id: a.id }, { $set: doc }, { upsert: true }); + upserted++; + } + console.log(`[LookupApp] Upserted ${upserted} applications`); + return { success: true, count: upserted }; + } catch (err) { + console.error('[LookupApp] Population error:', err.message); + return { success: false, count: 0 }; + } +} + module.exports = { collectAllAgents, collectSpecificAgent, - collectSpecificAgents + collectSpecificAgents, + populateLookupApps }; diff --git a/proxy/scheduler.js b/proxy/scheduler.js index e1f14b1..2995f4c 100644 --- a/proxy/scheduler.js +++ b/proxy/scheduler.js @@ -3,7 +3,7 @@ // Runs every 5 minutes, collecting data for all agents or a specific agent. const cron = require('node-cron'); -const { collectAllAgents, collectSpecificAgent, collectSpecificAgents } = require('./collector'); +const { collectAllAgents, collectSpecificAgent, collectSpecificAgents, populateLookupApps } = require('./collector'); // Mode: 'all' = collect all agents, 'agent' = collect one specific agent, 'agents' = collect multiple agents const COLLECT_MODE = process.env.PROXY_COLLECT_MODE || 'all'; @@ -21,6 +21,7 @@ let lastRunAt = null; let lastRunResult = null; let runCount = 0; let lastCapacityLogAt = 0; +let lastAppLookupRefresh = 0; /** * Execute one collection cycle (called by cron and manual trigger). @@ -58,6 +59,14 @@ async function runCollection() { const { logCapacityStats } = require('./db/capacityTracker'); await logCapacityStats(`[PROXY] [MongoDB] Capacity Used after Run #${runCount}:`); } + + // Refresh application lookup catalog daily (expensive: fetches 5000+ apps) + if (now - (lastAppLookupRefresh || 0) >= CAPACITY_LOG_INTERVAL_MS) { + lastAppLookupRefresh = now; + await populateLookupApps('').catch(err => + console.error('[Scheduler] LookupApp refresh failed:', err.message) + ); + } return lastRunResult; } catch (err) { @@ -96,8 +105,8 @@ function startScheduler() { // Initial run immediately on startup (async, do not block server start) setTimeout(async () => { - // 1. Sync dictionary first - // await netifyClient.syncApplicationDictionary(); + // 1. Sync application lookup catalog from DPI API + populateLookupApps('').catch(err => console.error('[Scheduler] Initial LookupApp sync failed:', err.message)); // 2. Start normal telemetry collection runCollection().catch(err => console.error('[Scheduler] Initial run error:', err.message)); -- 2.54.0