const path = require('path');
require('dotenv').config({ path: path.join(__dirname, '..', '.env') });
const express = require('express');
const cors = require('cors');
const db = require('./db');
const prometheusService = require('./prometheus-service');
const cache = require('./cache');
const geoService = require('./geo-service');
const latencyService = require('./latency-service');
const checkAndFixDatabase = require('./db-schema-check');
const http = require('http');
const WebSocket = require('ws');
const net = require('net');

const app = express();
const PORT = process.env.PORT || 3000;
const HOST = process.env.HOST || '0.0.0.0';

app.use(cors());
app.use(express.json());
const fs = require('fs');
const crypto = require('crypto');

let isDbInitialized = false;
const sessions = new Map(); // Fallback session store when Valkey is unavailable
const requestBuckets = new Map();
const SESSION_TTL_SECONDS = parseInt(process.env.SESSION_TTL_SECONDS, 10) || 86400;
const PASSWORD_ITERATIONS = parseInt(process.env.PASSWORD_ITERATIONS, 10) || 210000;
const ALLOW_REMOTE_SETUP = process.env.ALLOW_REMOTE_SETUP === 'true';
const COOKIE_SECURE = process.env.COOKIE_SECURE === 'true';
const APP_SECRET = process.env.APP_SECRET || crypto.randomBytes(32).toString('hex');
const RATE_LIMITS = {
  login: { windowMs: 15 * 60 * 1000, max: 8 },
  setup: { windowMs: 10 * 60 * 1000, max: 20 }
};

function normalizeIp(ip) {
  if (!ip) return '';
  if (ip.startsWith('::ffff:')) return ip.substring(7);
  return ip;
}

function isPrivateOrLoopbackIp(ip) {
  const normalized = normalizeIp(ip);

  if (!normalized) return false;
  if (normalized === '::1' || normalized === '127.0.0.1' || normalized === 'localhost') return true;

  if (net.isIPv4(normalized)) {
    return (
      normalized.startsWith('10.') ||
      normalized.startsWith('127.') ||
      normalized.startsWith('192.168.') ||
      /^172\.(1[6-9]|2\d|3[0-1])\./.test(normalized) ||
      normalized.startsWith('169.254.')
    );
  }

  if (net.isIPv6(normalized)) {
    const lower = normalized.toLowerCase();
    return lower === '::1' || lower.startsWith('fc') || lower.startsWith('fd') || lower.startsWith('fe80:');
  }

  return false;
}

function isLocalRequest(req) {
  const ip = normalizeIp(req.socket.remoteAddress || req.ip || '');
  return isPrivateOrLoopbackIp(ip);
}

function getClientIp(req) {
  return normalizeIp(req.socket?.remoteAddress || req.ip || '');
}

function makeRateLimitKey(scope, req, discriminator = '') {
  return `${scope}:${getClientIp(req)}:${discriminator}`;
}

function checkRateLimit(key, { windowMs, max }) {
  const now = Date.now();
  const bucket = requestBuckets.get(key);

  if (!bucket || bucket.resetAt <= now) {
    requestBuckets.set(key, { count: 1, resetAt: now + windowMs });
    return { allowed: true, retryAfterMs: 0 };
  }

  if (bucket.count >= max) {
    return {
      allowed: false,
      retryAfterMs: Math.max(0, bucket.resetAt - now)
    };
  }

  bucket.count += 1;
  return { allowed: true, retryAfterMs: 0 };
}

function enforceRateLimit(req, res, scope, discriminator = '') {
  const result = checkRateLimit(makeRateLimitKey(scope, req, discriminator), RATE_LIMITS[scope]);
  if (result.allowed) {
    return true;
  }

  res.setHeader('Retry-After', Math.max(1, Math.ceil(result.retryAfterMs / 1000)));
  res.status(429).json({
    success: false,
    error: 'Too many requests. Please try again later.'
  });
  return false;
}

function escapeJsonForInlineScript(value) {
  return JSON.stringify(value).replace(/[<>&\u2028\u2029]/g, (char) => {
    switch (char) {
      case '<':
        return '\\u003c';
      case '>':
        return '\\u003e';
      case '&':
        return '\\u0026';
      case '\u2028':
        return '\\u2028';
      case '\u2029':
        return '\\u2029';
      default:
        return char;
    }
  });
}

function getPublicSiteSettings(settings = {}) {
  return {
    page_name: settings.page_name || '',
    show_page_name: settings.show_page_name !== undefined ? settings.show_page_name : 1,
    title: settings.title || '',
    logo_url: settings.logo_url || null,
    logo_url_dark: settings.logo_url_dark || null,
    favicon_url: settings.favicon_url || null,
    default_theme: settings.default_theme || 'dark',
    show_95_bandwidth: settings.show_95_bandwidth ? 1 : 0,
    p95_type: settings.p95_type || 'tx',
    require_login_for_server_details: settings.require_login_for_server_details !== undefined
      ? (settings.require_login_for_server_details ? 1 : 0)
      : 1,
    icp_filing: settings.icp_filing || null,
    ps_filing: settings.ps_filing || null,
    network_data_sources: settings.network_data_sources || null,
    show_server_ip: settings.show_server_ip ? 1 : 0,
    ip_metric_name: settings.ip_metric_name || null,
    ip_label_name: settings.ip_label_name || 'address',
    custom_metrics: settings.custom_metrics || []
  };
}

async function getSiteSettingsRow() {
  const [rows] = await db.query('SELECT * FROM site_settings WHERE id = 1');
  return rows.length > 0 ? rows[0] : {};
}

async function requireServerDetailsAccess(req, res, next) {
  try {
    const settings = await getSiteSettingsRow();
    req.siteSettings = settings; // Store for later use (e.g. IP stripping)
    
    const requiresLogin = settings.require_login_for_server_details !== undefined
      ? !!settings.require_login_for_server_details
      : true;

    if (!requiresLogin) {
      return next();
    }

    return requireAuth(req, res, next);
  } catch (err) {
    console.error('Server details access check failed:', err);
    return res.status(500).json({ error: 'Failed to verify detail access' });
  }
}

function getCookieOptions(req, maxAgeSeconds) {
  const options = ['Path=/', 'HttpOnly', 'SameSite=Strict'];
  if (typeof maxAgeSeconds === 'number') {
    options.push(`Max-Age=${maxAgeSeconds}`);
  }
  if (COOKIE_SECURE || req.secure || req.headers['x-forwarded-proto'] === 'https') {
    options.push('Secure');
  }
  return options.join('; ');
}

function setSessionCookie(req, res, sessionId, maxAgeSeconds = SESSION_TTL_SECONDS) {
  res.setHeader('Set-Cookie', `session_id=${encodeURIComponent(sessionId)}; ${getCookieOptions(req, maxAgeSeconds)}`);
}

function clearSessionCookie(req, res) {
  res.setHeader('Set-Cookie', `session_id=; ${getCookieOptions(req, 0)}`);
}

function createPasswordHash(password, salt, iterations = PASSWORD_ITERATIONS) {
  const hash = crypto.pbkdf2Sync(password, salt, iterations, 64, 'sha512').toString('hex');
  return `pbkdf2$sha512$${iterations}$${hash}`;
}

function parsePasswordHash(storedPassword) {
  if (typeof storedPassword !== 'string') {
    return { iterations: 1000, hash: '' };
  }

  if (storedPassword.startsWith('pbkdf2$')) {
    const parts = storedPassword.split('$');
    if (parts.length === 4 && parts[1] === 'sha512') {
      return {
        iterations: parseInt(parts[2], 10) || 1000,
        hash: parts[3]
      };
    }
  }

  return { iterations: 1000, hash: storedPassword };
}

function verifyPassword(password, user) {
  const parsed = parsePasswordHash(user.password);
  const computed = crypto.pbkdf2Sync(password, user.salt, parsed.iterations, 64, 'sha512').toString('hex');

  try {
    return crypto.timingSafeEqual(Buffer.from(computed, 'hex'), Buffer.from(parsed.hash, 'hex'));
  } catch (err) {
    return false;
  }
}

function shouldUpgradePasswordHash(user) {
  const parsed = parsePasswordHash(user.password);
  return !String(user.password || '').startsWith('pbkdf2$') || parsed.iterations < PASSWORD_ITERATIONS;
}

async function persistSession(sessionId, sessionData) {
  sessions.set(sessionId, {
    ...sessionData,
    expiresAt: Date.now() + SESSION_TTL_SECONDS * 1000
  });
  await cache.set(`session:${sessionId}`, sessionData, SESSION_TTL_SECONDS);
}

async function getSession(sessionId) {
  if (!sessionId) return null;

  const cachedSession = await cache.get(`session:${sessionId}`);
  if (cachedSession) {
    sessions.set(sessionId, {
      ...cachedSession,
      expiresAt: Date.now() + SESSION_TTL_SECONDS * 1000
    });
    return cachedSession;
  }

  const fallback = sessions.get(sessionId);
  if (!fallback) return null;
  if (fallback.expiresAt && fallback.expiresAt <= Date.now()) {
    sessions.delete(sessionId);
    return null;
  }

  return {
    id: fallback.id,
    username: fallback.username
  };
}

async function destroySession(sessionId) {
  if (!sessionId) return;
  sessions.delete(sessionId);
  await cache.del(`session:${sessionId}`);
}

async function ensureSetupAccess(req, res, next) {
  if (!enforceRateLimit(req, res, 'setup')) {
    return;
  }
  if (ALLOW_REMOTE_SETUP || isLocalRequest(req)) {
    return next();
  }
  return res.status(403).json({
    success: false,
    error: 'Remote setup is disabled. Use a local request or set ALLOW_REMOTE_SETUP=true.'
  });
}

// Middleware: Check Auth
async function requireAuth(req, res, next) {
  const sessionId = getCookie(req, 'session_id');
  const session = await getSession(sessionId);
  if (session) {
    req.user = session;
    return next();
  }
  res.status(401).json({ error: 'Auth required' });
}

// Helper: Get Cookie
function getCookie(req, name) {
  const matches = req.headers.cookie && req.headers.cookie.match(new RegExp('(?:^|; )' + name.replace(/([\.$?*|{}\(\)\[\]\\\/\+^])/g, '\\$1') + '=([^;]*)'));
  return matches ? decodeURIComponent(matches[1]) : undefined;
}

/**
 * Database Initialization Check
 */
async function checkDb() {
  try {
    const fs = require('fs');
    // In Docker or high-level environments, .env might not exist but process.env is set
    const hasConfig = process.env.MYSQL_HOST || fs.existsSync(path.join(__dirname, '..', '.env'));
    
    if (!hasConfig) {
      isDbInitialized = false;
      return;
    }

    // Check for essential tables
    const [rows] = await db.query("SHOW TABLES LIKE 'prometheus_sources'");
    const [rows2] = await db.query("SHOW TABLES LIKE 'site_settings'");
    
    isDbInitialized = rows.length > 0 && rows2.length > 0;
  } catch (err) {
    isDbInitialized = false;
  }
}

checkDb();

// --- Health API ---
app.get('/health', async (req, res) => {
  try {
    const dbStatus = await db.checkHealth();
    const cacheStatus = await cache.checkHealth();
    const isAllOk = dbStatus.status === 'up' && cacheStatus.status === 'up';

    const healthInfo = {
      status: isAllOk ? 'ok' : 'error',
      timestamp: new Date().toISOString(),
      service: {
        status: 'running',
        uptime: Math.floor(process.uptime()),
        memory_usage: {
          rss: Math.floor(process.memoryUsage().rss / 1024 / 1024) + ' MB',
          heapTotal: Math.floor(process.memoryUsage().heapTotal / 1024 / 1024) + ' MB'
        },
        node_version: process.version
      },
      checks: {
        database: {
          name: 'MySQL',
          status: dbStatus.status,
          message: dbStatus.status === 'up' ? 'Connected' : 'Unavailable'
        },
        valkey: {
          name: 'Valkey (Redis)',
          status: cacheStatus.status,
          message: cacheStatus.status === 'up' ? 'Connected' : 'Unavailable'
        }
      }
    };

    if (isAllOk) {
      res.json(healthInfo);
    } else {
      res.status(500).json(healthInfo);
    }
  } catch (err) {
    res.status(500).json({ status: 'error', message: err.message });
  }
});

// --- Auth API ---
app.post('/api/auth/login', async (req, res) => {
  const { username, password } = req.body;
  const loginKey = String(username || '').trim().toLowerCase();
  if (!enforceRateLimit(req, res, 'login', loginKey)) {
    return;
  }
  try {
    const [rows] = await db.query('SELECT * FROM users WHERE username = ?', [username]);
    if (rows.length === 0) return res.status(401).json({ error: 'Invalid credentials' });
    
    const user = rows[0];
    
    if (verifyPassword(password, user)) {
      if (shouldUpgradePasswordHash(user)) {
        await db.query('UPDATE users SET password = ? WHERE id = ?', [createPasswordHash(password, user.salt), user.id]);
      }

      const sessionId = crypto.randomBytes(32).toString('hex');
      await persistSession(sessionId, { id: user.id, username: user.username });
      setSessionCookie(req, res, sessionId);
      res.json({ success: true, username: user.username });
    } else {
      res.status(401).json({ error: 'Invalid credentials' });
    }
  } catch (err) {
    res.status(500).json({ error: 'Login failed' });
  }
});

app.post('/api/auth/logout', (req, res) => {
  const sessionId = getCookie(req, 'session_id');
  destroySession(sessionId).catch(() => {});
  clearSessionCookie(req, res);
  res.json({ success: true });
});

app.post('/api/auth/change-password', requireAuth, async (req, res) => {
  const { oldPassword, newPassword } = req.body;
  if (!oldPassword || !newPassword) {
    return res.status(400).json({ error: '需要输入旧密码和新密码' });
  }

  try {
    const [rows] = await db.query('SELECT * FROM users WHERE id = ?', [req.user.id]);
    if (rows.length === 0) return res.status(404).json({ error: '用户不存在' });

    const user = rows[0];
    const passwordMatches = verifyPassword(oldPassword, user);

    if (!passwordMatches) {
      return res.status(401).json({ error: '旧密码输入错误' });
    }

    const newSalt = crypto.randomBytes(16).toString('hex');
    const newHash = createPasswordHash(newPassword, newSalt);

    await db.query('UPDATE users SET password = ?, salt = ? WHERE id = ?', [newHash, newSalt, user.id]);
    res.json({ success: true, message: '密码修改成功' });
  } catch (err) {
    console.error('Password update error:', err);
    res.status(500).json({ error: '服务器错误，修改失败' });
  }
});

app.get('/api/auth/status', async (req, res) => {
  const sessionId = getCookie(req, 'session_id');
  const session = await getSession(sessionId);
  if (session) {
    res.json({ authenticated: true, username: session.username });
  } else {
    res.json({ authenticated: false });
  }
});

// Setup API Routes
app.post('/api/setup/test', ensureSetupAccess, async (req, res) => {
  const { host, port, user, password } = req.body;
  try {
    const mysql = require('mysql2/promise');
    const connection = await mysql.createConnection({
      host: host || 'localhost',
      port: parseInt(port) || 3306,
      user: user || 'root',
      password: password || ''
    });
    await connection.ping();
    await connection.end();
    res.json({ success: true, message: 'Connection successful' });
  } catch (err) {
    res.status(400).json({ success: false, error: err.message });
  }
});

app.post('/api/setup/test-valkey', ensureSetupAccess, async (req, res) => {
  const { host, port, password } = req.body;
  try {
    const Redis = require('ioredis');
    const redis = new Redis({
      host: host || 'localhost',
      port: parseInt(port) || 6379,
      password: password || undefined,
      lazyConnect: true,
      maxRetriesPerRequest: 1,
      connectTimeout: 5000
    });
    await redis.connect();
    await redis.ping();
    await redis.disconnect();
    res.json({ success: true, message: 'Valkey connection successful' });
  } catch (err) {
    res.status(400).json({ success: false, error: err.message });
  }
});

app.post('/api/setup/init', ensureSetupAccess, async (req, res) => {
  const { host, port, user, password, database, vHost, vPort, vPassword } = req.body;
  try {
    if (isDbInitialized) {
      return res.status(409).json({ success: false, error: 'System is already initialized' });
    }

    const mysql = require('mysql2/promise');
    const connection = await mysql.createConnection({
      host: host || 'localhost',
      port: parseInt(port) || 3306,
      user: user || 'root',
      password: password || ''
    });

    const dbName = database || 'display_wall';

    // Create database
    await connection.query(`CREATE DATABASE IF NOT EXISTS \`${dbName}\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`);
    await connection.query(`USE \`${dbName}\``);

    // Create tables
    await connection.query(`
      CREATE TABLE IF NOT EXISTS prometheus_sources (
        id INT AUTO_INCREMENT PRIMARY KEY,
        name VARCHAR(255) NOT NULL,
        url VARCHAR(500) NOT NULL,
        description TEXT,
        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
        updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
      ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
    `);

    await connection.query(`
      CREATE TABLE IF NOT EXISTS users (
        id INT AUTO_INCREMENT PRIMARY KEY,
        username VARCHAR(255) NOT NULL UNIQUE,
        password VARCHAR(255) NOT NULL,
        salt VARCHAR(255) NOT NULL,
        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
      ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
    `);

    await connection.query(`
      CREATE TABLE IF NOT EXISTS traffic_stats (
        id INT AUTO_INCREMENT PRIMARY KEY,
        rx_bytes BIGINT UNSIGNED DEFAULT 0,
        tx_bytes BIGINT UNSIGNED DEFAULT 0,
        rx_bandwidth DOUBLE DEFAULT 0,
        tx_bandwidth DOUBLE DEFAULT 0,
        timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
        UNIQUE INDEX (timestamp)
      ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
    `);

    await connection.query(`
      CREATE TABLE IF NOT EXISTS site_settings (
        id INT PRIMARY KEY DEFAULT 1,
        page_name VARCHAR(255) DEFAULT '数据可视化展示大屏',
        title VARCHAR(255) DEFAULT '数据可视化展示大屏',
        logo_url TEXT,
        logo_url_dark TEXT,
        favicon_url TEXT,
        default_theme VARCHAR(20) DEFAULT 'dark',
        show_95_bandwidth TINYINT(1) DEFAULT 0,
        p95_type VARCHAR(20) DEFAULT 'tx',
        require_login_for_server_details TINYINT(1) DEFAULT 1,
        blackbox_source_id INT,
        latency_source VARCHAR(100),
        latency_dest VARCHAR(100),
        latency_target VARCHAR(255),
        icp_filing VARCHAR(255),
        ps_filing VARCHAR(255),
        network_data_sources TEXT,
        show_server_ip TINYINT(1) DEFAULT 0,
        ip_metric_name VARCHAR(100) DEFAULT NULL,
        ip_label_name VARCHAR(100) DEFAULT 'address',
        custom_metrics JSON DEFAULT NULL,
        updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
      ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
    `);
    await connection.query(`
      INSERT IGNORE INTO site_settings (id, page_name, title, default_theme, show_95_bandwidth, p95_type) 
      VALUES (1, '数据可视化展示大屏', '数据可视化展示大屏', 'dark', 0, 'tx')
    `);

    // Note: checkAndFixDatabase (called later in this route) will handle column migrations correctly and compatibly.
    await connection.query(`
      CREATE TABLE IF NOT EXISTS latency_routes (
        id INT AUTO_INCREMENT PRIMARY KEY,
        source_id INT NOT NULL,
        latency_source VARCHAR(100) NOT NULL,
        latency_dest VARCHAR(100) NOT NULL,
        latency_target VARCHAR(255) NOT NULL,
        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
      ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
    `);
    await connection.query(`
      CREATE TABLE IF NOT EXISTS server_locations (
        id INT AUTO_INCREMENT PRIMARY KEY,
        ip VARCHAR(255) NOT NULL UNIQUE,
        country CHAR(2),
        country_name VARCHAR(100),
        region VARCHAR(100),
        city VARCHAR(100),
        latitude DOUBLE,
        longitude DOUBLE,
        last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
      ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
    `);

    await connection.end();

    // Save to .env
    const envContent = `MYSQL_HOST=${host || 'localhost'}
MYSQL_PORT=${port || '3306'}
MYSQL_USER=${user || 'root'}
MYSQL_PASSWORD=${password || ''}
MYSQL_DATABASE=${dbName}
VALKEY_HOST=${vHost || 'localhost'}
VALKEY_PORT=${vPort || '6379'}
VALKEY_PASSWORD=${vPassword || ''}
PORT=${process.env.PORT || 3000}
HOST=${process.env.HOST || '0.0.0.0'}
REFRESH_INTERVAL=${process.env.REFRESH_INTERVAL || 5000}
ALLOW_REMOTE_SETUP=${process.env.ALLOW_REMOTE_SETUP || 'false'}
COOKIE_SECURE=${process.env.COOKIE_SECURE || 'false'}
SESSION_TTL_SECONDS=${process.env.SESSION_TTL_SECONDS || SESSION_TTL_SECONDS}
PASSWORD_ITERATIONS=${process.env.PASSWORD_ITERATIONS || PASSWORD_ITERATIONS}
ENABLE_EXTERNAL_GEO_LOOKUP=${process.env.ENABLE_EXTERNAL_GEO_LOOKUP || 'false'}
APP_SECRET=${process.env.APP_SECRET || APP_SECRET}
`;
    fs.writeFileSync(path.join(__dirname, '..', '.env'), envContent);
    
    // Update process.env
    process.env.MYSQL_HOST = host;
    process.env.MYSQL_PORT = port;
    process.env.MYSQL_USER = user;
    process.env.MYSQL_PASSWORD = password;
    process.env.MYSQL_DATABASE = dbName;
    process.env.VALKEY_HOST = vHost;
    process.env.VALKEY_PORT = vPort;
    process.env.VALKEY_PASSWORD = vPassword;

    // Re-initialize pools
    db.initPool();
    cache.init();

    // Run the migration/centralized schema tool to create/fix all tables
    await checkAndFixDatabase();

    isDbInitialized = true;
    res.json({ success: true, message: 'Initialization complete' });
  } catch (err) {
    console.error('Initialization error:', err);
    res.status(500).json({ success: false, error: err.message });
  }
});

// Setup Status Check
app.get('/api/setup/status', ensureSetupAccess, async (req, res) => {
  try {
    if (!isDbInitialized) {
      return res.json({ initialized: false, step: 'db' });
    }
    const [rows] = await db.query('SELECT COUNT(*) as count FROM users');
    if (rows[0].count === 0) {
      return res.json({ initialized: true, needsAdmin: true, step: 'admin' });
    }
    res.json({ initialized: true, needsAdmin: false, step: 'prom' });
  } catch (err) {
    console.error('Status check error:', err);
    res.json({ initialized: false, step: 'db' });
  }
});

// Create First Admin
app.post('/api/setup/admin', ensureSetupAccess, async (req, res) => {
  const { username, password } = req.body;
  if (!username || !password) return res.status(400).json({ error: 'Username and password are required' });
  
  try {
    const [rows] = await db.query('SELECT COUNT(*) as count FROM users');
    if (rows[0].count > 0) return res.status(403).json({ error: 'Admin already exists' });
    
    const salt = crypto.randomBytes(16).toString('hex');
    const hash = createPasswordHash(password, salt);
    
    await db.query('INSERT INTO users (username, password, salt) VALUES (?, ?, ?)', [username, hash, salt]);
    const [userRows] = await db.query('SELECT id, username FROM users WHERE username = ?', [username]);
    const user = userRows[0];

    // Auto-login after creation so the next setup steps (like adding Prometheus) work without 401
    const sessionId = crypto.randomBytes(32).toString('hex');
    await persistSession(sessionId, { id: user.id, username: user.username });
    setSessionCookie(req, res, sessionId);

    res.json({ success: true, message: 'Admin account created and logged in' });
  } catch (err) {
    console.error('Admin creation error:', err);
    res.status(500).json({ error: err.message });
  }
});

// Middleware to protect routes & enforce setup
app.use(async (req, res, next) => {
  // Allow system files and setup APIs
  if (req.path === '/health' || req.path.startsWith('/api/setup') || req.path === '/init.html' || req.path.startsWith('/css/') || req.path.startsWith('/js/') || req.path.startsWith('/fonts/')) {
    return next();
  }

  // Enforce DB setup
  if (!isDbInitialized) {
    if (req.path.startsWith('/api/')) {
      return res.status(503).json({ error: 'Database not initialized', needSetup: true });
    }
    return res.redirect('/init.html');
  }

  // Enforce User setup
  try {
    const [rows] = await db.query('SELECT COUNT(*) as count FROM users');
    if (rows[0].count === 0) {
      if (req.path.startsWith('/api/')) {
        return res.status(503).json({ error: 'Admin not configured', needAdmin: true });
      }
      return res.redirect('/init.html?step=admin');
    }
  } catch (err) {
    // If table doesn't exist, it's a DB initialization issue
  }

  if (req.path === '/init.html') {
    return res.redirect('/');
  }
  next();
});

// Helper to serve index.html with injected settings
const serveIndex = async (req, res) => {
  try {
    const indexPath = path.join(__dirname, '..', 'public', 'index.html');
    if (!fs.existsSync(indexPath)) return res.status(404).send('Not found');
    
    let html = fs.readFileSync(indexPath, 'utf8');
    
    // Fetch settings
    let settings = { 
      page_name: '数据可视化展示大屏', 
      show_page_name: 1,
      title: '数据可视化展示大屏', 
      logo_url: null,
      logo_url_dark: null,
      favicon_url: null, 
      default_theme: 'dark',
      blackbox_source_id: null,
      latency_source: null,
      latency_dest: null,
      latency_target: null,
      icp_filing: null,
      ps_filing: null,
      network_data_sources: null
    };
    
    if (isDbInitialized) {
      try {
        const [rows] = await db.query('SELECT * FROM site_settings WHERE id = 1');
        if (rows.length > 0) settings = rows[0];
      } catch (e) {
        // DB not ready or table missing
      }
    }

    // Inject settings
    const settingsJson = escapeJsonForInlineScript(getPublicSiteSettings(settings));
    const injection = `<script>window.SITE_SETTINGS = ${settingsJson};</script>`;
    
    // Replace <head> with <head> + injection
    html = html.replace('<head>', '<head>' + injection);
    
    res.send(html);
  } catch (err) {
    console.error('Error serving index:', err);
    res.status(500).send('Internal Server Error');
  }
};

app.get('/', serveIndex);
app.get('/index.html', serveIndex);

app.use(express.static(path.join(__dirname, '..', 'public'), { index: false }));

// ==================== Prometheus Source CRUD ====================

// Get all Prometheus sources
app.get('/api/sources', requireAuth, async (req, res) => {
  try {
    const [rows] = await db.query('SELECT * FROM prometheus_sources ORDER BY is_server_source DESC, created_at DESC');
    // Test connectivity for each source
    const sourcesWithStatus = await Promise.all(rows.map(async (source) => {
      try {
        let response;
        if (source.type === 'blackbox') {
          // Simple check for blackbox exporter
          const res = await fetch(`${source.url.replace(/\/+$/, '')}/metrics`, { timeout: 3000 }).catch(() => null);
          response = (res && res.ok) ? 'Blackbox Exporter Ready' : 'Connection Error';
        } else {
          response = await prometheusService.testConnection(source.url);
        }
        return { ...source, status: 'online', version: response };
      } catch (e) {
        return { ...source, status: 'offline', version: null };
      }
    }));
    res.json(sourcesWithStatus);
  } catch (err) {
    console.error('Error fetching sources:', err);
    res.status(500).json({ error: 'Failed to fetch sources' });
  }
});

// Add a new Prometheus source
app.post('/api/sources', requireAuth, async (req, res) => {
  let { name, url, description, is_server_source, type } = req.body;
  if (!name || !url) {
    return res.status(400).json({ error: 'Name and URL are required' });
  }
  if (!/^https?:\/\//i.test(url)) url = 'http://' + url;
  try {
    const [result] = await db.query(
      'INSERT INTO prometheus_sources (name, url, description, is_server_source, type) VALUES (?, ?, ?, ?, ?)',
      [name, url, description || '', is_server_source === undefined ? 1 : (is_server_source ? 1 : 0), type || 'prometheus']
    );
    const [rows] = await db.query('SELECT * FROM prometheus_sources WHERE id = ?', [result.insertId]);
    
    // Clear network history cache to force refresh
    await cache.del('network_history_all');

    res.status(201).json(rows[0]);
  } catch (err) {
    console.error('Error adding source:', err);
    res.status(500).json({ error: 'Failed to add source' });
  }
});

// Update a Prometheus source
app.put('/api/sources/:id', requireAuth, async (req, res) => {
  let { name, url, description, is_server_source, type } = req.body;
  if (url && !/^https?:\/\//i.test(url)) url = 'http://' + url;
  try {
    await db.query(
      'UPDATE prometheus_sources SET name = ?, url = ?, description = ?, is_server_source = ?, type = ? WHERE id = ?',
      [name, url, description || '', is_server_source ? 1 : 0, type || 'prometheus', req.params.id]
    );
    // Clear network history cache
    await cache.del('network_history_all');

    const [rows] = await db.query('SELECT * FROM prometheus_sources WHERE id = ?', [req.params.id]);
    res.json(rows[0]);
  } catch (err) {
    console.error('Error updating source:', err);
    res.status(500).json({ error: 'Failed to update source' });
  }
});

// Delete a Prometheus source
app.delete('/api/sources/:id', requireAuth, async (req, res) => {
  try {
    await db.query('DELETE FROM prometheus_sources WHERE id = ?', [req.params.id]);
    // Clear network history cache
    await cache.del('network_history_all');
    res.json({ message: 'Source deleted' });
  } catch (err) {
    console.error('Error deleting source:', err);
    res.status(500).json({ error: 'Failed to delete source' });
  }
});

// Test connection to a Prometheus source
app.post('/api/sources/test', requireAuth, async (req, res) => {
  let { url, type } = req.body;
  if (url && !/^https?:\/\//i.test(url)) url = 'http://' + url;
  try {
    let result;
    if (type === 'blackbox') {
      const resVal = await fetch(`${url.replace(/\/+$/, '')}/metrics`, { timeout: 5000 }).catch(() => null);
      result = (resVal && resVal.ok) ? 'Blackbox Exporter Ready' : 'Connection Failed';
      if (!resVal || !resVal.ok) throw new Error(result);
    } else {
      result = await prometheusService.testConnection(url);
    }
    res.json({ status: 'ok', version: result });
  } catch (err) {
    res.status(400).json({ status: 'error', message: err.message });
  }
});

// ==================== Site Settings ====================

// Get site settings
app.get('/api/settings', async (req, res) => {
  try {
    const [rows] = await db.query('SELECT * FROM site_settings WHERE id = 1');
    if (rows.length === 0) {
      return res.json(getPublicSiteSettings());
    }
    return res.json(getPublicSiteSettings(rows[0]));
  } catch (err) {
    console.error('Error fetching settings:', err);
    res.status(500).json({ error: 'Failed to fetch settings' });
  }
});

// Update site settings
app.post('/api/settings', requireAuth, async (req, res) => {
  try {
    // 1. Fetch current settings first to preserve fields not sent by the UI
    const [rows] = await db.query('SELECT * FROM site_settings WHERE id = 1');
    let current = rows.length > 0 ? rows[0] : {};

    // 2. Destructure fields from body
    const { 
      page_name, show_page_name, title, logo_url, logo_url_dark, favicon_url, 
      default_theme, show_95_bandwidth, p95_type, require_login_for_server_details,
      icp_filing, ps_filing, network_data_sources, show_server_ip, ip_metric_name, ip_label_name, custom_metrics
    } = req.body;

    // 3. Prepare parameters, prioritizing body but falling back to current
    const settings = {
      page_name: page_name !== undefined ? page_name : (current.page_name || '数据可视化展示大屏'),
      show_page_name: show_page_name !== undefined ? (show_page_name ? 1 : 0) : (current.show_page_name !== undefined ? current.show_page_name : 1),
      title: title !== undefined ? title : (current.title || '数据可视化展示大屏'),
      logo_url: logo_url !== undefined ? logo_url : (current.logo_url || null),
      logo_url_dark: logo_url_dark !== undefined ? logo_url_dark : (current.logo_url_dark || null),
      favicon_url: favicon_url !== undefined ? favicon_url : (current.favicon_url || null),
      default_theme: default_theme !== undefined ? default_theme : (current.default_theme || 'dark'),
      show_95_bandwidth: show_95_bandwidth !== undefined ? (show_95_bandwidth ? 1 : 0) : (current.show_95_bandwidth || 0),
      p95_type: p95_type !== undefined ? p95_type : (current.p95_type || 'tx'),
      require_login_for_server_details: require_login_for_server_details !== undefined
        ? (require_login_for_server_details ? 1 : 0)
        : (current.require_login_for_server_details !== undefined ? current.require_login_for_server_details : 1),
      blackbox_source_id: current.blackbox_source_id || null,
      latency_source: current.latency_source || null,
      latency_dest: current.latency_dest || null,
      latency_target: current.latency_target || null,
      icp_filing: icp_filing !== undefined ? icp_filing : (current.icp_filing || null),
      ps_filing: ps_filing !== undefined ? ps_filing : (current.ps_filing || null),
      network_data_sources: network_data_sources !== undefined ? network_data_sources : (current.network_data_sources || null),
      show_server_ip: show_server_ip !== undefined ? (show_server_ip ? 1 : 0) : (current.show_server_ip || 0),
      ip_metric_name: ip_metric_name !== undefined ? ip_metric_name : (current.ip_metric_name || null),
      ip_label_name: ip_label_name !== undefined ? ip_label_name : (current.ip_label_name || 'address'),
      custom_metrics: custom_metrics !== undefined ? JSON.stringify(custom_metrics) : (current.custom_metrics || '[]')
    };

    await db.query(`
      INSERT INTO site_settings (
        id, page_name, show_page_name, title, logo_url, logo_url_dark, favicon_url, 
        default_theme, show_95_bandwidth, p95_type, require_login_for_server_details,
        blackbox_source_id, latency_source, latency_dest, latency_target, 
        icp_filing, ps_filing, network_data_sources, show_server_ip, ip_metric_name, ip_label_name,
        custom_metrics
      ) VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) 
      ON DUPLICATE KEY UPDATE 
        page_name = VALUES(page_name), 
        show_page_name = VALUES(show_page_name),
        title = VALUES(title), 
        logo_url = VALUES(logo_url),
        logo_url_dark = VALUES(logo_url_dark),
        favicon_url = VALUES(favicon_url),
        default_theme = VALUES(default_theme),
        show_95_bandwidth = VALUES(show_95_bandwidth),
        p95_type = VALUES(p95_type),
        require_login_for_server_details = VALUES(require_login_for_server_details),
        blackbox_source_id = VALUES(blackbox_source_id),
        latency_source = VALUES(latency_source),
        latency_dest = VALUES(latency_dest),
        latency_target = VALUES(latency_target),
        icp_filing = VALUES(icp_filing),
        ps_filing = VALUES(ps_filing),
        network_data_sources = VALUES(network_data_sources),
        show_server_ip = VALUES(show_server_ip),
        ip_metric_name = VALUES(ip_metric_name),
        ip_label_name = VALUES(ip_label_name),
        custom_metrics = VALUES(custom_metrics)`,
      [
        settings.page_name, settings.show_page_name, settings.title, settings.logo_url, settings.logo_url_dark, settings.favicon_url,
        settings.default_theme, settings.show_95_bandwidth, settings.p95_type, settings.require_login_for_server_details,
        settings.blackbox_source_id, settings.latency_source, settings.latency_dest, settings.latency_target,
        settings.icp_filing, settings.ps_filing, settings.network_data_sources, settings.show_server_ip,
        settings.ip_metric_name, settings.ip_label_name, settings.custom_metrics
      ]
    );

    res.json({ success: true, settings: getPublicSiteSettings(settings) });
  } catch (err) {
    console.error('Error updating settings:', err);
    res.status(500).json({ error: 'Failed to update settings' });
  }
});

// ==================== Metrics Aggregation ====================

// Reusable function to get overview metrics
async function getOverview(force = false) {
  const [sources] = await db.query('SELECT * FROM prometheus_sources WHERE is_server_source = 1 AND type != "blackbox"');
  if (sources.length === 0) {
    return {
      totalServers: 0,
      activeServers: 0,
      cpu: { used: 0, total: 0, percent: 0 },
      memory: { used: 0, total: 0, percent: 0 },
      disk: { used: 0, total: 0, percent: 0 },
      network: { total: 0, rx: 0, tx: 0 },
      traffic24h: { rx: 0, tx: 0, total: 0 },
      servers: []
    };
  }

  const [settingsRows] = await db.query('SELECT network_data_sources FROM site_settings WHERE id = 1');
  const selectedSourcesStr = settingsRows.length > 0 ? settingsRows[0].network_data_sources : null;
  const selectedSourceNames = selectedSourcesStr ? selectedSourcesStr.split(',').map(s => s.trim()).filter(s => s) : [];

  const allMetrics = await Promise.all(sources.map(async (source) => {
    const cacheKey = `source_metrics:${source.url}:${source.name}`;
    if (force) {
      await cache.del(cacheKey);
    } else {
      const cached = await cache.get(cacheKey);
      if (cached) return cached;
    }

    try {
      const metrics = await prometheusService.getOverviewMetrics(source.url, source.name);
      const enrichedMetrics = { ...metrics, sourceName: source.name };
      
      await cache.set(cacheKey, enrichedMetrics, 15); // Cache for 15s
      return enrichedMetrics;
    } catch (err) {
      console.error(`Error fetching metrics from ${source.name}:`, err.message);
      return null;
    }
  }));

  const validMetrics = allMetrics.filter(m => m !== null);

  // Aggregate across all sources
  let totalServers = 0;
  let activeServers = 0;
  let cpuUsed = 0, cpuTotal = 0;
  let memUsed = 0, memTotal = 0;
  let diskUsed = 0, diskTotal = 0;
  let netRx = 0, netTx = 0;
  let traffic24hRx = 0, traffic24hTx = 0;
  let allServers = [];

  for (const m of validMetrics) {
    totalServers += m.totalServers;
    activeServers += (m.activeServers !== undefined ? m.activeServers : m.totalServers);
    cpuUsed += m.cpu.used;
    cpuTotal += m.cpu.total;
    memUsed += m.memory.used;
    memTotal += m.memory.total;
    diskUsed += m.disk.used;
    diskTotal += m.disk.total;
    
    // Aggregates ONLY for selected network sources
    if (selectedSourceNames.length === 0 || selectedSourceNames.includes(m.sourceName)) {
      netRx += m.network.rx;
      netTx += m.network.tx;
      traffic24hRx += m.traffic24h.rx;
      traffic24hTx += m.traffic24h.tx;
    }

    allServers = allServers.concat(m.servers);
  }

  const overview = {
    totalServers,
    activeServers,
    cpu: {
      used: cpuUsed,
      total: cpuTotal,
      percent: cpuTotal > 0 ? (cpuUsed / cpuTotal * 100) : 0
    },
    memory: {
      used: memUsed,
      total: memTotal,
      percent: memTotal > 0 ? (memUsed / memTotal * 100) : 0
    },
    disk: {
      used: diskUsed,
      total: diskTotal,
      percent: diskTotal > 0 ? (diskUsed / diskTotal * 100) : 0
    },
    network: {
      total: netRx + netTx,
      rx: netRx,
      tx: netTx
    },
    traffic24h: {
      rx: traffic24hRx,
      tx: traffic24hTx,
      total: traffic24hRx + traffic24hTx
    },
    servers: allServers
  };

  // --- Add Geo Information to Servers ---
  const geoServers = await Promise.all(overview.servers.map(async (server) => {
    const realInstance = server.originalInstance || prometheusService.resolveToken(server.instance);
    // Helper to get host from instance (handles IPv6 with brackets, IPv4:port, etc.)
    let cleanIp = realInstance;
    if (cleanIp.startsWith('[')) {
      const closingBracket = cleanIp.indexOf(']');
      if (closingBracket !== -1) {
        cleanIp = cleanIp.substring(1, closingBracket);
      }
    } else {
      const parts = cleanIp.split(':');
      // If exactly one colon, it's likely IPv4:port or host:port
      if (parts.length === 2) {
        cleanIp = parts[0];
      }
      // If more than 1 colon and no brackets, it's an IPv6 without port - keep as is
    }
    
    let geoData = null;
    try {
      const [rows] = await db.query('SELECT * FROM server_locations WHERE ip = ?', [cleanIp]);
      if (rows.length > 0) {
        geoData = rows[0];
      } else {
        geoService.getLocation(cleanIp).catch(() => {});
      }
    } catch (e) {}

    const { originalInstance, ...safeServer } = server;
    if (geoData) {
      return {
        ...safeServer,
        country: geoData.country,
        countryName: geoData.country_name,
        city: geoData.city,
        lat: geoData.latitude,
        lng: geoData.longitude
      };
    }
    return safeServer;
  }));

  overview.servers = geoServers;
  return overview;
}

// Get all aggregated metrics from all Prometheus sources
app.get('/api/metrics/overview', async (req, res) => {
  try {
    const force = req.query.force === 'true';
    const overview = await getOverview(force);
    res.json(overview);
  } catch (err) {
    console.error('Error fetching overview metrics:', err);
    res.status(500).json({ error: 'Failed to fetch metrics' });
  }
});

// Get network traffic history (past 24h) from Prometheus
app.get('/api/metrics/network-history', async (req, res) => {
  try {
    const force = req.query.force === 'true';
    const cacheKey = 'network_history_all';
    
    if (force) {
      await cache.del(cacheKey);
    } else {
      const cached = await cache.get(cacheKey);
      if (cached) return res.json(cached);
    }

    const [settingsRows] = await db.query('SELECT network_data_sources FROM site_settings WHERE id = 1');
    const selectedSourcesStr = settingsRows.length > 0 ? settingsRows[0].network_data_sources : null;
    
    let query = 'SELECT * FROM prometheus_sources WHERE is_server_source = 1 AND type != "blackbox"';
    let params = [];
    
    if (selectedSourcesStr) {
      const selectedSourceNames = selectedSourcesStr.split(',').map(s => s.trim()).filter(s => s);
      if (selectedSourceNames.length > 0) {
        query += ' AND name IN (?)';
        params.push(selectedSourceNames);
      }
    }

    const [sources] = await db.query(query, params);
    if (sources.length === 0) {
      return res.json({ timestamps: [], rx: [], tx: [] });
    }

    const histories = await Promise.all(sources.map(source => 
      prometheusService.getNetworkHistory(source.url).catch(err => {
        console.error(`Error fetching network history from ${source.name}:`, err.message);
        return null;
      })
    ));

    const validHistories = histories.filter(h => h !== null);
    if (validHistories.length === 0) {
      return res.json({ timestamps: [], rx: [], tx: [] });
    }

    const merged = prometheusService.mergeNetworkHistories(validHistories);
    await cache.set(cacheKey, merged, 300); // Cache for 5 minutes
    res.json(merged);
  } catch (err) {
    console.error('Error fetching network history history:', err);
    res.status(500).json({ error: 'Failed to fetch network history history' });
  }
});

// Get CPU usage history for sparklines
app.get('/api/metrics/cpu-history', async (req, res) => {
  try {
    const [sources] = await db.query('SELECT * FROM prometheus_sources WHERE is_server_source = 1 AND type != "blackbox"');
    if (sources.length === 0) {
      return res.json({ timestamps: [], values: [] });
    }

    const allHistories = await Promise.all(sources.map(source =>
      prometheusService.getCpuHistory(source.url).catch(err => {
        console.error(`Error fetching CPU history from ${source.name}:`, err.message);
        return null;
      })
    ));

    const validHistories = allHistories.filter(h => h !== null);
    if (validHistories.length === 0) {
      return res.json({ timestamps: [], values: [] });
    }

    const merged = prometheusService.mergeCpuHistories(validHistories);
    res.json(merged);
  } catch (err) {
    console.error('Error fetching CPU history:', err);
    res.status(500).json({ error: 'Failed to fetch CPU history' });
  }
});

// Get detailed metrics for a specific server
app.get('/api/metrics/server-details', requireServerDetailsAccess, async (req, res) => {
  const { instance, job, source } = req.query;
  
  if (!instance || !job || !source) {
    return res.status(400).json({ error: 'instance, job, and source name are required' });
  }

  try {
    // Find the source URL by name
    const [rows] = await db.query('SELECT url FROM prometheus_sources WHERE name = ?', [source]);
    if (rows.length === 0) {
      return res.status(404).json({ error: 'Prometheus source not found' });
    }
    const sourceUrl = rows[0].url;

    // Fetch detailed metrics with custom metric configuration if present
    const details = await prometheusService.getServerDetails(sourceUrl, instance, job, req.siteSettings);

    // Dynamic field removal based on security settings: PHYSICAL DATA STRIPPING
    if (!req.siteSettings || !req.siteSettings.show_server_ip) {
      delete details.ipv4;
      delete details.ipv6;
    }

    res.json(details);
  } catch (err) {
    console.error(`Error fetching server details for ${instance}:`, err.message);
    res.status(500).json({ error: 'Failed to fetch server details' });
  }
});

// Get historical metrics for a specific server
app.get('/api/metrics/server-history', requireServerDetailsAccess, async (req, res) => {
  const { instance, job, source, metric, range, start, end } = req.query;
  
  if (!instance || !job || !source || !metric) {
    return res.status(400).json({ error: 'instance, job, source, and metric are required' });
  }
  try {
    const [rows] = await db.query('SELECT url FROM prometheus_sources WHERE name = ?', [source]);
    if (rows.length === 0) return res.status(404).json({ error: 'Source not found' });
    const sourceUrl = rows[0].url;

    // Fetch p95Type from settings for networkTrend stats
    let p95Type = 'tx';
    try {
        const [settingsRows] = await db.query('SELECT p95_type FROM site_settings WHERE id = 1');
        if (settingsRows.length > 0) p95Type = settingsRows[0].p95_type;
    } catch (e) {}

    const data = await prometheusService.getServerHistory(sourceUrl, instance, job, metric, range, start, end, p95Type);
    res.json(data);
  } catch (err) {
    res.status(500).json({ error: err.message });
  }
});

// ==================== Latency Routes CRUD ====================

app.get('/api/latency-routes', requireAuth, async (req, res) => {
  try {
    const [rows] = await db.query(`
      SELECT r.*, s.name as source_name 
      FROM latency_routes r 
      LEFT JOIN prometheus_sources s ON r.source_id = s.id 
      ORDER BY r.created_at DESC
    `);
    res.json(rows);
  } catch (err) {
    res.status(500).json({ error: 'Failed to fetch latency routes' });
  }
});

app.post('/api/latency-routes', requireAuth, async (req, res) => {
  const { source_id, latency_source, latency_dest, latency_target } = req.body;
  try {
    await db.query('INSERT INTO latency_routes (source_id, latency_source, latency_dest, latency_target) VALUES (?, ?, ?, ?)', [source_id, latency_source, latency_dest, latency_target]);
    res.json({ success: true });
  } catch (err) {
    res.status(500).json({ error: 'Failed to add latency route' });
  }
});

app.delete('/api/latency-routes/:id', requireAuth, async (req, res) => {
  try {
    await db.query('DELETE FROM latency_routes WHERE id = ?', [req.params.id]);
    res.json({ success: true });
  } catch (err) {
    res.status(500).json({ error: 'Failed to delete latency route' });
  }
});

app.put('/api/latency-routes/:id', requireAuth, async (req, res) => {
  const { source_id, latency_source, latency_dest, latency_target } = req.body;
  try {
    await db.query(
      'UPDATE latency_routes SET source_id = ?, latency_source = ?, latency_dest = ?, latency_target = ? WHERE id = ?',
      [source_id, latency_source, latency_dest, latency_target, req.params.id]
    );
    res.json({ success: true });
  } catch (err) {
    res.status(500).json({ error: 'Failed to update latency route' });
  }
});

// ==================== Metrics Latency ====================

app.get('/api/metrics/latency', async (req, res) => {
  try {
    const [routes] = await db.query(`
      SELECT r.*, s.url, s.type as source_type
      FROM latency_routes r 
      JOIN prometheus_sources s ON r.source_id = s.id
    `);
    
    if (routes.length === 0) {
      return res.json({ routes: [] });
    }

    const results = await Promise.all(routes.map(async (route) => {
      // Try to get from Valkey first (filled by background latencyService)
      let latency = await cache.get(`latency:route:${route.id}`);
      
      // Fallback if not in cache (only for prometheus sources, blackbox sources rely on the background service)
      if (latency === null && route.source_type === 'prometheus') {
        latency = await prometheusService.getLatency(route.url, route.latency_target);
      }

      return {
        id: route.id,
        source: route.latency_source,
        dest: route.latency_dest,
        latency: latency
      };
    }));

    res.json({ routes: results });
  } catch (err) {
    console.error('Error fetching latencies:', err);
    res.status(500).json({ error: 'Failed to fetch latency' });
  }
});

// ==================== WebSocket Server ====================

const server = http.createServer(app);
const wss = new WebSocket.Server({ server });
let isBroadcastRunning = false;
let cachedLatencyRoutes = null;
let lastRoutesUpdate = 0;

function broadcast(data) {
  const message = JSON.stringify(data);
  wss.clients.forEach(client => {
    if (client.readyState === WebSocket.OPEN) {
      client.send(message);
    }
  });
}

async function broadcastMetrics() {
  if (isBroadcastRunning) return;
  isBroadcastRunning = true;
  try {
    const overview = await getOverview();
    
    // Refresh routes list every 60 seconds or if it hasn't been fetched yet
    const now = Date.now();
    if (!cachedLatencyRoutes || now - lastRoutesUpdate > 60000) {
      const [routes] = await db.query(`
        SELECT r.*, s.url, s.type as source_type
        FROM latency_routes r 
        JOIN prometheus_sources s ON r.source_id = s.id
      `);
      cachedLatencyRoutes = routes;
      lastRoutesUpdate = now;
    }
    
    const latencyResults = await Promise.all(cachedLatencyRoutes.map(async (route) => {
      let latency = await cache.get(`latency:route:${route.id}`);
      if (latency === null && route.source_type === 'prometheus') {
        latency = await prometheusService.getLatency(route.url, route.latency_target);
      }
      return {
        id: route.id,
        source: route.latency_source,
        dest: route.latency_dest,
        latency: latency
      };
    }));

    broadcast({ 
      type: 'overview', 
      data: {
        ...overview,
        latencies: latencyResults
      }
    });
  } catch (err) {
    // console.error('WS Broadcast error:', err.message);
  } finally {
    isBroadcastRunning = false;
  }
}

// Start server and services
async function start() {
  try {
    console.log('🔧 Initializing services...');
    
    // 1. Initial check
    await checkDb();

    // 2. Automated repair/migration
    try {
      const dbFixed = await checkAndFixDatabase();
      if (dbFixed) {
        // Refresh state after fix
        await checkDb();
        if (isDbInitialized) {
          console.log('  ✅ Database integrity verified and initialized');
        }
      }
    } catch (dbErr) {
      console.error('❌ Critical database initialization error:', dbErr.message);
      // If we have an .env but can't connect, this is a fatal config error
      if (fs.existsSync(path.join(__dirname, '..', '.env'))) {
        console.error('  Please check your MYSQL settings in .env or run setup wizard.');
        // Don't exit, allow the user to reach the init/setup page to fix configurations
      }
    }
    
    // Start services
    latencyService.start();

    const REFRESH_INT = parseInt(process.env.REFRESH_INTERVAL) || 5000;
    setInterval(broadcastMetrics, REFRESH_INT);

    // Periodic cleanup of rate limit buckets to prevent memory leak
    setInterval(() => {
      const now = Date.now();
      for (const [key, bucket] of requestBuckets.entries()) {
        if (bucket.resetAt <= now) {
          requestBuckets.delete(key);
        }
      }
    }, 3600000); // Once per hour

    // Periodic cleanup of sessions Map to prevent memory growth
    setInterval(() => {
      const now = Date.now();
      for (const [sessionId, session] of sessions.entries()) {
        if (session.expiresAt && session.expiresAt <= now) {
          sessions.delete(sessionId);
        }
      }
    }, 300000); // Once every 5 minutes

    server.listen(PORT, HOST, () => {
      console.log(`\n  🚀 Data Visualization Display Wall (WebSocket Enabled)`);
      console.log(`  📊 Server running at http://${HOST === '0.0.0.0' ? 'localhost' : HOST}:${PORT}`);
      console.log(`  ⚙️  Configure Prometheus sources at http://${HOST === '0.0.0.0' ? 'localhost' : HOST}:${PORT}/settings\n`);
    });
  } catch (err) {
    console.error('❌ Server failed to start:', err.message);
    process.exit(1);
  }
}

process.on('unhandledRejection', (reason, promise) => {
  console.error('[System] Unhandled Rejection at:', promise, 'reason:', reason);
});

process.on('uncaughtException', (err) => {
  console.error('[System] Uncaught Exception:', err);
});

start();