Update collector to not use authentication

proxy/collector.js

@@ -18,6 +18,8 @@ const {
 } = require('./collectorHelperDpi2');
 
 const { Summary, AppStat } = require('./models/Schemas');
+const { LookupApp } = require('./models/SchemasAux');
+const { BASE_URL } = require('./netifyClientCore');
 const { pruneOldData } = require('./dataRetention');
 
 const SITE_UUIDS_STR = process.env.NETIFY_SITE_UUIDS || process.env.NETIFY_SITE_UUID;
@@ -255,8 +257,47 @@ async function collectSpecificAgents(agentUuids, siteUuid = SITE_UUIDS[0], delay
   return { success: true, mode: 'specific_agents', agents_count: agentUuids.length, successful, results };
 }
 
+async function populateLookupApps(siteUuid = '') {
+  try {
+    const axios = require('axios');
+    const headers = { 'Accept': 'application/json' };
+    const res = await axios.get(`${BASE_URL}/lookup/applications`, {
+      headers, params: { settings_limit: 10000 }, timeout: 20000
+    });
+    const apps = res.data?.data;
+    if (!apps || !Array.isArray(apps)) {
+      console.log('[LookupApp] No application data from DPI API');
+      return { success: false, count: 0 };
+    }
+    let upserted = 0;
+    for (const a of apps) {
+      if (!a.id) continue;
+      const doc = {
+        id: a.id,
+        tag: a.tag || '',
+        label: a.label || '',
+        name: a.full_label || a.label || '',
+        full_name: a.full_label || '',
+        description: a.description || '',
+        favicon: a.favicon || '',
+        icon: a.icon || '',
+        logo: a.logo || '',
+        application_category: a.category || null
+      };
+      await LookupApp.updateOne({ id: a.id }, { $set: doc }, { upsert: true });
+      upserted++;
+    }
+    console.log(`[LookupApp] Upserted ${upserted} applications`);
+    return { success: true, count: upserted };
+  } catch (err) {
+    console.error('[LookupApp] Population error:', err.message);
+    return { success: false, count: 0 };
+  }
+}
+
 module.exports = {
   collectAllAgents,
   collectSpecificAgent,
-  collectSpecificAgents
+  collectSpecificAgents,
+  populateLookupApps
 };

proxy/scheduler.js

@@ -3,7 +3,7 @@
 // Runs every 5 minutes, collecting data for all agents or a specific agent.
 
 const cron = require('node-cron');
-const { collectAllAgents, collectSpecificAgent, collectSpecificAgents } = require('./collector');
+const { collectAllAgents, collectSpecificAgent, collectSpecificAgents, populateLookupApps } = require('./collector');
 
 // Mode: 'all' = collect all agents, 'agent' = collect one specific agent, 'agents' = collect multiple agents
 const COLLECT_MODE    = process.env.PROXY_COLLECT_MODE || 'all';
@@ -21,6 +21,7 @@ let lastRunAt = null;
 let lastRunResult = null;
 let runCount = 0;
 let lastCapacityLogAt = 0;
+let lastAppLookupRefresh = 0;
 
 /**
  * Execute one collection cycle (called by cron and manual trigger).
@@ -58,6 +59,14 @@ async function runCollection() {
       const { logCapacityStats } = require('./db/capacityTracker');
       await logCapacityStats(`[PROXY] [MongoDB] Capacity Used after Run #${runCount}:`);
     }
+
+    // Refresh application lookup catalog daily (expensive: fetches 5000+ apps)
+    if (now - (lastAppLookupRefresh || 0) >= CAPACITY_LOG_INTERVAL_MS) {
+      lastAppLookupRefresh = now;
+      await populateLookupApps('').catch(err =>
+        console.error('[Scheduler] LookupApp refresh failed:', err.message)
+      );
+    }
     
     return lastRunResult;
   } catch (err) {
@@ -96,8 +105,8 @@ function startScheduler() {
   
   // Initial run immediately on startup (async, do not block server start)
   setTimeout(async () => {
-    // 1. Sync dictionary first
+    // 1. Sync application lookup catalog from DPI API
-    // await netifyClient.syncApplicationDictionary();
+    populateLookupApps('').catch(err => console.error('[Scheduler] Initial LookupApp sync failed:', err.message));
     
     // 2. Start normal telemetry collection
     runCollection().catch(err => console.error('[Scheduler] Initial run error:', err.message));