chore: automate production deployment and safe database migration scripts

scripts/deploy-sftp.js

@@ -37,6 +37,7 @@ const UPLOAD_MANIFEST = [
   { local: '.next/standalone', remote: '.next/standalone', type: 'dir' },
   { local: '.next/static', remote: '.next/static', type: 'dir' },
   { local: 'public', remote: 'public', type: 'dir' },
+  { local: 'public/api/uploads', remote: 'api/uploads', type: 'dir' },
 
   // Backend API server
   { local: 'backend/server.js', remote: 'backend/server.js', type: 'file' },
@@ -55,12 +56,14 @@ const UPLOAD_MANIFEST = [
   { local: 'proxy/collectorHelper.js', remote: 'proxy/collectorHelper.js', type: 'file' },
   { local: 'proxy/collectorHelperDpi.js', remote: 'proxy/collectorHelperDpi.js', type: 'file' },
   { local: 'proxy/collectorHelperDpi2.js', remote: 'proxy/collectorHelperDpi2.js', type: 'file' },
+  { local: 'proxy/collectorHelperDpi3.js', remote: 'proxy/collectorHelperDpi3.js', type: 'file' },
   { local: 'proxy/dataRetention.js', remote: 'proxy/dataRetention.js', type: 'file' },
   { local: 'proxy/db', remote: 'proxy/db', type: 'dir' },
   { local: 'proxy/deviceResolver.js', remote: 'proxy/deviceResolver.js', type: 'file' },
   { local: 'proxy/models', remote: 'proxy/models', type: 'dir' },
   { local: 'proxy/netifyClient.js', remote: 'proxy/netifyClient.js', type: 'file' },
   { local: 'proxy/netifyClientCore.js', remote: 'proxy/netifyClientCore.js', type: 'file' },
+  { local: 'proxy/netifyClientStats.js', remote: 'proxy/netifyClientStats.js', type: 'file' },
   { local: 'proxy/netifyTelemetry.js', remote: 'proxy/netifyTelemetry.js', type: 'file' },
   { local: 'proxy/routes.js', remote: 'proxy/routes.js', type: 'file' },
   { local: 'proxy/scheduler.js', remote: 'proxy/scheduler.js', type: 'file' },
@@ -70,6 +73,10 @@ const UPLOAD_MANIFEST = [
   // PM2 ecosystem config
   { local: 'ecosystem.config.js', remote: 'ecosystem.config.js', type: 'file' },
 
+  // Database migration temporary files
+  { local: 'migration-temp', remote: 'migration-temp', type: 'dir' },
+  { local: 'scripts/migrate-db-remote.js', remote: 'migration-temp/migrate-db-remote.js', type: 'file' },
+
   // Production environment (CONFIDENTIAL — uploaded via SFTP, not git)
   { local: '.env.production', remote: '.env.production', type: 'file' },
 ];
@@ -93,6 +100,11 @@ const POST_DEPLOY_COMMANDS = [
   // Install proxy dependencies (production only)
   'cd /home/adminbackend/web/demoplace.my.id/public_html/proxy && npm ci --omit=dev 2>&1 | tail -5',
 
+  // Execute database migration on remote server
+  'echo "=== RUNNING REMOTE DB MIGRATION ==="',
+  'node /home/adminbackend/web/demoplace.my.id/public_html/migration-temp/migrate-db-remote.js',
+  'rm -rf /home/adminbackend/web/demoplace.my.id/public_html/migration-temp && echo "Cleaned up migration temporary files"',
+
   // Copy Next.js static assets to standalone directory (required for standalone mode)
   // Fix standalone directory structure for Next.js
   'mkdir -p /home/adminbackend/web/demoplace.my.id/public_html/.next/standalone/.next/',

scripts/export-local-db.js

@@ -0,0 +1,42 @@
+// scripts/export-local-db.js
+const { MongoClient } = require('mongodb');
+const fs = require('fs');
+const path = require('path');
+
+const LOCAL_URI = 'mongodb://127.0.0.1:27017/backone_dpi';
+const DB_NAME = 'backone_dpi';
+
+const COLLECTIONS_TO_EXPORT = ['users', 'tenantconfigs', 'customagentlocations'];
+const OUTPUT_DIR = path.join(__dirname, '..', 'migration-temp');
+
+async function main() {
+  console.log('Connecting to local MongoDB...');
+  const client = new MongoClient(LOCAL_URI);
+  
+  try {
+    await client.connect();
+    console.log('✓ Connected to local MongoDB successfully.');
+    const db = client.db(DB_NAME);
+    
+    if (!fs.existsSync(OUTPUT_DIR)) {
+      fs.mkdirSync(OUTPUT_DIR);
+    }
+    
+    for (const colName of COLLECTIONS_TO_EXPORT) {
+      console.log(`Exporting collection: ${colName}...`);
+      const documents = await db.collection(colName).find({}).toArray();
+      const filePath = path.join(OUTPUT_DIR, `${colName}.json`);
+      fs.writeFileSync(filePath, JSON.stringify(documents, null, 2));
+      console.log(`  ✓ Exported ${documents.length} documents to ${filePath}`);
+    }
+    
+    console.log('\n✓ Database export complete!');
+  } catch (err) {
+    console.error('✗ Export failed:', err.message);
+    process.exit(1);
+  } finally {
+    await client.close();
+  }
+}
+
+main();

scripts/migrate-db-remote.js

@@ -0,0 +1,83 @@
+// scripts/migrate-db-remote.js
+global.crypto = require('crypto');
+const { MongoClient } = require('mongodb');
+const fs = require('fs');
+const path = require('path');
+
+const REMOTE_URI = 'mongodb://backone_user:SusuKudaLiar@103.80.237.29:27017/backone_dpi?authSource=backone_dpi';
+const DB_NAME = 'backone_dpi';
+const DATA_DIR = __dirname;
+
+async function migrateCollection(db, colName, keyField) {
+  const filePath = path.join(DATA_DIR, `${colName}.json`);
+  if (!fs.existsSync(filePath)) {
+    console.log(`  [SKIP] ${colName}.json not found in migration folder.`);
+    return;
+  }
+
+  const raw = fs.readFileSync(filePath, 'utf8');
+  const documents = JSON.parse(raw);
+  console.log(`\nMigrating collection: ${colName} (${documents.length} docs)...`);
+
+  const col = db.collection(colName);
+  let upsertedCount = 0;
+  let matchedCount = 0;
+
+  for (const doc of documents) {
+    // Preserve MongoDB Object ID or convert it properly if it has $oid notation
+    const query = {};
+    query[keyField] = doc[keyField];
+
+    // Clean up _id or preserve it
+    const updateDoc = { ...doc };
+    if (updateDoc._id) {
+      if (updateDoc._id.$oid) {
+        // If it's exported via mongoexport format
+        delete updateDoc._id;
+      } else {
+        // Remove _id from update payload to prevent "change _id" errors on upsert
+        delete updateDoc._id;
+      }
+    }
+
+    const result = await col.updateOne(
+      query,
+      { $set: updateDoc },
+      { upsert: true }
+    );
+
+    if (result.upsertedCount > 0) {
+      upsertedCount++;
+    } else {
+      matchedCount++;
+    }
+  }
+
+  console.log(`  ✓ Finished ${colName}: ${upsertedCount} new inserted (upserted), ${matchedCount} existing updated.`);
+}
+
+async function main() {
+  console.log('=== STARTING PRODUCTION DATABASE MIGRATION ===');
+  console.log('Connecting to production MongoDB...');
+  const client = new MongoClient(REMOTE_URI, { serverSelectionTimeoutMS: 5000 });
+
+  try {
+    await client.connect();
+    console.log('✓ Connected to production MongoDB successfully!');
+    const db = client.db(DB_NAME);
+
+    // Run migrations using distinct key fields for upserting
+    await migrateCollection(db, 'users', 'username');
+    await migrateCollection(db, 'tenantconfigs', 'site_uuid');
+    await migrateCollection(db, 'customagentlocations', 'agent_uuid');
+
+    console.log('\n=== MIGRATION COMPLETED SUCCESSFULLY ===');
+  } catch (err) {
+    console.error('\n✗ Migration failed:', err.message);
+    process.exit(1);
+  } finally {
+    await client.close();
+  }
+}
+
+main();