const axios = require('axios'); const http = require('http'); const https = require('https'); const QUERY_TIMEOUT = 10000; // Reusable agents to handle potential redirect issues and protocol mismatches const crypto = require('crypto'); const httpAgent = new http.Agent({ keepAlive: true }); const httpsAgent = new https.Agent({ keepAlive: true, rejectUnauthorized: false }); const serverIdMap = new Map(); // token -> { instance, job, source } const SECRET = crypto.randomBytes(16).toString('hex'); function getServerToken(instance) { const hash = crypto.createHmac('sha256', SECRET) .update(instance) .digest('hex') .substring(0, 16); return hash; } /** * Normalize URL and ensure protocol */ function normalizeUrl(baseUrl) { let url = baseUrl.trim().replace(/\/+$/, ''); if (!/^https?:\/\//i.test(url)) { url = 'http://' + url; } return url; } /** * Create an axios instance for a given Prometheus URL */ function createClient(baseUrl) { const url = normalizeUrl(baseUrl); return axios.create({ baseURL: url, timeout: QUERY_TIMEOUT, httpAgent, httpsAgent, maxRedirects: 5 }); } /** * Test Prometheus connection */ async function testConnection(url) { const normalized = normalizeUrl(url); try { // Using native fetch to avoid follow-redirects/axios "protocol mismatch" issues in some Node environments const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), QUERY_TIMEOUT); // Node native fetch - handles http/https automatically const res = await fetch(`${normalized}/api/v1/status/buildinfo`, { signal: controller.signal }); clearTimeout(timer); if (!res.ok) { throw new Error(`Prometheus returned HTTP ${res.status}`); } const data = await res.json(); return data?.data?.version || 'unknown'; } catch (err) { if (err.name === 'AbortError') { throw new Error('Connection timed out'); } console.error(`[Prometheus] Connection test failed for ${normalized}:`, err.message); throw err; } } /** * Execute a Prometheus instant query */ async function query(baseUrl, expr) { const url = normalizeUrl(baseUrl); try { const params = new URLSearchParams({ query: expr }); const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), QUERY_TIMEOUT); const res = await fetch(`${url}/api/v1/query?${params.toString()}`, { signal: controller.signal }); clearTimeout(timer); if (!res.ok) { throw new Error(`Prometheus returned HTTP ${res.status}`); } const data = await res.json(); if (data.status !== 'success') { throw new Error(`Prometheus query failed: ${data.error || 'unknown error'}`); } return data.data.result; } catch (err) { if (err.name === 'AbortError') { throw new Error('Prometheus query timed out'); } throw err; } } /** * Get all targets from Prometheus */ async function getTargets(baseUrl) { const url = normalizeUrl(baseUrl); try { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), QUERY_TIMEOUT); const res = await fetch(`${url}/api/v1/targets`, { signal: controller.signal }); clearTimeout(timer); if (!res.ok) { throw new Error(`Prometheus returned HTTP ${res.status}`); } const data = await res.json(); if (data.status !== 'success') { throw new Error(`Prometheus targets fetch failed: ${data.error || 'unknown error'}`); } return data.data.activeTargets || []; } catch (err) { if (err.name === 'AbortError') { throw new Error('Prometheus targets fetch timed out'); } throw err; } } /** * Execute a Prometheus range query */ async function queryRange(baseUrl, expr, start, end, step) { const url = normalizeUrl(baseUrl); try { const params = new URLSearchParams({ query: expr, start, end, step }); const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), QUERY_TIMEOUT); const res = await fetch(`${url}/api/v1/query_range?${params.toString()}`, { signal: controller.signal }); clearTimeout(timer); if (!res.ok) { throw new Error(`Prometheus returned HTTP ${res.status}`); } const data = await res.json(); if (data.status !== 'success') { throw new Error(`Prometheus range query failed: ${data.error || 'unknown error'}`); } return data.data.result; } catch (err) { if (err.name === 'AbortError') { throw new Error('Prometheus range query timed out'); } throw err; } } async function getOverviewMetrics(url, sourceName) { // Run all queries in parallel const [ cpuResult, cpuCountResult, memTotalResult, memAvailResult, diskTotalResult, diskFreeResult, netRxResult, netTxResult, traffic24hRxResult, traffic24hTxResult, targetsResult ] = await Promise.all([ // CPU usage per instance: 1 - avg idle query(url, '100 - (avg by (instance, job) (rate(node_cpu_seconds_total{mode="idle"}[1m])) * 100)').catch(() => []), // CPU count per instance query(url, 'count by (instance, job) (node_cpu_seconds_total{mode="idle"})').catch(() => []), // Memory total per instance query(url, 'node_memory_MemTotal_bytes').catch(() => []), // Memory available per instance query(url, 'node_memory_MemAvailable_bytes').catch(() => []), // Disk total per instance (root filesystem + /data) query(url, 'sum by (instance, job) (node_filesystem_size_bytes{mountpoint=~"/|/data",fstype!="tmpfs"})').catch(() => []), // Disk free per instance (root filesystem + /data) query(url, 'sum by (instance, job) (node_filesystem_free_bytes{mountpoint=~"/|/data",fstype!="tmpfs"})').catch(() => []), // Network receive rate (bytes/sec) query(url, 'sum by (instance, job) (rate(node_network_receive_bytes_total{device!~"lo|veth.*|docker.*|br-.*"}[1m]))').catch(() => []), // Network transmit rate (bytes/sec) query(url, 'sum by (instance, job) (rate(node_network_transmit_bytes_total{device!~"lo|veth.*|docker.*|br-.*"}[1m]))').catch(() => []), // Total traffic received in last 24h query(url, 'sum by (instance, job) (increase(node_network_receive_bytes_total{device!~"lo|veth.*|docker.*|br-.*"}[24h]))').catch(() => []), // Total traffic transmitted in last 24h query(url, 'sum by (instance, job) (increase(node_network_transmit_bytes_total{device!~"lo|veth.*|docker.*|br-.*"}[24h]))').catch(() => []), // Targets status from /api/v1/targets getTargets(url).catch(() => []) ]); // Build per-instance data map const instances = new Map(); const getOrCreate = (metric) => { const originalInstance = metric.instance; const token = getServerToken(originalInstance, sourceName); // Store mapping for detail queries serverIdMap.set(token, { instance: originalInstance, source: sourceName, job: metric.job }); if (!instances.has(token)) { instances.set(token, { instance: token, // This is the masked IP SENT TO FRONTEND originalInstance, // Keep internal for aggregation/parsing job: metric.job || 'Unknown', source: sourceName, cpuPercent: 0, cpuCores: 0, memTotal: 0, memUsed: 0, diskTotal: 0, diskUsed: 0, netRx: 0, netTx: 0, up: false }); } const inst = instances.get(token); // If job was Unknown but we now have a job name, update it if (inst.job === 'Unknown' && metric.job) { inst.job = metric.job; } return inst; }; // Initialize instances from targets first (to ensure we have all servers even if they have no metrics) const nodeJobRegex = /node|exporter|host/i; for (const target of targetsResult) { const labels = target.labels || {}; const instance = labels.instance; const job = labels.job; // Only include targets that look like node-exporters if (instance && (nodeJobRegex.test(job) || nodeJobRegex.test(target.scrapePool))) { const inst = getOrCreate(labels); inst.up = target.health === 'up'; } } // Parse CPU usage for (const r of cpuResult) { const inst = getOrCreate(r.metric); inst.cpuPercent = parseFloat(r.value[1]) || 0; } // Parse CPU count for (const r of cpuCountResult) { const inst = getOrCreate(r.metric); inst.cpuCores = parseFloat(r.value[1]) || 0; } // Parse memory for (const r of memTotalResult) { const inst = getOrCreate(r.metric); inst.memTotal = parseFloat(r.value[1]) || 0; } for (const r of memAvailResult) { const inst = getOrCreate(r.metric); inst.memUsed = inst.memTotal - (parseFloat(r.value[1]) || 0); } // Parse disk for (const r of diskTotalResult) { const inst = getOrCreate(r.metric); inst.diskTotal = parseFloat(r.value[1]) || 0; } for (const r of diskFreeResult) { const inst = getOrCreate(r.metric); inst.diskUsed = inst.diskTotal - (parseFloat(r.value[1]) || 0); } // Parse network rates for (const r of netRxResult) { const inst = getOrCreate(r.metric); inst.netRx = parseFloat(r.value[1]) || 0; } for (const r of netTxResult) { const inst = getOrCreate(r.metric); inst.netTx = parseFloat(r.value[1]) || 0; } for (const inst of instances.values()) { if (!inst.up && (inst.cpuPercent > 0 || inst.memTotal > 0)) { inst.up = true; } } const allInstancesList = Array.from(instances.values()); const activeInstances = allInstancesList.filter(inst => inst.up); // Aggregate let totalCpuUsed = 0, totalCpuCores = 0; let totalMemUsed = 0, totalMemTotal = 0; let totalDiskUsed = 0, totalDiskTotal = 0; let totalNetRx = 0, totalNetTx = 0; let totalTraffic24hRx = 0, totalTraffic24hTx = 0; for (const inst of activeInstances) { totalCpuUsed += (inst.cpuPercent / 100) * inst.cpuCores; totalCpuCores += inst.cpuCores; totalMemUsed += inst.memUsed; totalMemTotal += inst.memTotal; totalDiskUsed += inst.diskUsed; totalDiskTotal += inst.diskTotal; totalNetRx += inst.netRx; totalNetTx += inst.netTx; } // Parse 24h traffic for (const r of traffic24hRxResult) { totalTraffic24hRx += parseFloat(r.value[1]) || 0; } for (const r of traffic24hTxResult) { totalTraffic24hTx += parseFloat(r.value[1]) || 0; } return { totalServers: allInstancesList.length, activeServers: activeInstances.length, cpu: { used: totalCpuUsed, total: totalCpuCores, percent: totalCpuCores > 0 ? (totalCpuUsed / totalCpuCores * 100) : 0 }, memory: { used: totalMemUsed, total: totalMemTotal, percent: totalMemTotal > 0 ? (totalMemUsed / totalMemTotal * 100) : 0 }, disk: { used: totalDiskUsed, total: totalDiskTotal, percent: totalDiskTotal > 0 ? (totalDiskUsed / totalDiskTotal * 100) : 0 }, network: { rx: totalNetRx, tx: totalNetTx, total: totalNetRx + totalNetTx }, traffic24h: { rx: totalTraffic24hRx, tx: totalTraffic24hTx, total: totalTraffic24hRx + totalTraffic24hTx }, servers: allInstancesList.map(s => { const { originalInstance, ...rest } = s; return rest; }) }; } /** * Get network traffic history (past 24h, 15-min intervals) */ async function getNetworkHistory(url) { const now = Math.floor(Date.now() / 1000); const start = now - 86400; // 24h ago const step = 900; // 15 minutes const [rxResult, txResult] = await Promise.all([ queryRange(url, 'sum(rate(node_network_receive_bytes_total{device!~"lo|veth.*|docker.*|br-.*"}[1m]))', start, now, step ).catch(() => []), queryRange(url, 'sum(rate(node_network_transmit_bytes_total{device!~"lo|veth.*|docker.*|br-.*"}[1m]))', start, now, step ).catch(() => []) ]); // Extract values - each result[0].values = [[timestamp, value], ...] const rxValues = rxResult.length > 0 ? rxResult[0].values : []; const txValues = txResult.length > 0 ? txResult[0].values : []; return { rxValues, txValues }; } /** * Merge network histories from multiple sources */ function mergeNetworkHistories(histories) { const timestampMap = new Map(); for (const history of histories) { for (const [ts, val] of history.rxValues) { const existing = timestampMap.get(ts) || { rx: 0, tx: 0 }; existing.rx += parseFloat(val) || 0; timestampMap.set(ts, existing); } for (const [ts, val] of history.txValues) { const existing = timestampMap.get(ts) || { rx: 0, tx: 0 }; existing.tx += parseFloat(val) || 0; timestampMap.set(ts, existing); } } const sorted = [...timestampMap.entries()].sort((a, b) => a[0] - b[0]); return { timestamps: sorted.map(([ts]) => ts * 1000), // ms for JS rx: sorted.map(([, v]) => v.rx), tx: sorted.map(([, v]) => v.tx) }; } /** * Get CPU usage history (past 1h, 1-min intervals) */ async function getCpuHistory(url) { const now = Math.floor(Date.now() / 1000); const start = now - 3600; // 1h ago const step = 60; // 1 minute const result = await queryRange(url, '100 - (avg(rate(node_cpu_seconds_total{mode="idle"}[1m])) * 100)', start, now, step ).catch(() => []); const values = result.length > 0 ? result[0].values : []; return values; // [[timestamp, value], ...] } /** * Merge CPU histories from multiple sources (average) */ function mergeCpuHistories(histories) { const timestampMap = new Map(); for (const history of histories) { for (const [ts, val] of history) { const existing = timestampMap.get(ts) || { sum: 0, count: 0 }; existing.sum += parseFloat(val) || 0; existing.count += 1; timestampMap.set(ts, existing); } } const sorted = [...timestampMap.entries()].sort((a, b) => a[0] - b[0]); return { timestamps: sorted.map(([ts]) => ts * 1000), values: sorted.map(([, v]) => v.sum / v.count) }; } function resolveToken(token) { if (serverIdMap.has(token)) { return serverIdMap.get(token).instance; } return token; } /** * Get detailed metrics for a specific server (node) */ async function getServerDetails(baseUrl, instance, job) { const url = normalizeUrl(baseUrl); const node = resolveToken(instance); // Queries based on the requested dashboard structure const queries = { // Split CPU cpuSystem: `avg(rate(node_cpu_seconds_total{mode="system", instance="${node}"}[1m])) * 100`, cpuUser: `avg(rate(node_cpu_seconds_total{mode="user", instance="${node}"}[1m])) * 100`, cpuIowait: `avg(rate(node_cpu_seconds_total{mode="iowait", instance="${node}"}[1m])) * 100`, cpuIrq: `avg(rate(node_cpu_seconds_total{mode=~"irq|softirq", instance="${node}"}[1m])) * 100`, cpuOther: `avg(rate(node_cpu_seconds_total{mode=~"nice|steal|guest|guest_nice", instance="${node}"}[1m])) * 100`, cpuIdle: `avg(rate(node_cpu_seconds_total{mode="idle", instance="${node}"}[1m])) * 100`, cpuBusy: `100 * (1 - avg(rate(node_cpu_seconds_total{mode="idle", instance="${node}"}[1m])))`, sysLoad: `node_load1{instance="${node}",job="${job}"} * 100 / count(count(node_cpu_seconds_total{instance="${node}",job="${job}"}) by (cpu))`, memUsedPct: `(1 - (node_memory_MemAvailable_bytes{instance="${node}", job="${job}"} / node_memory_MemTotal_bytes{instance="${node}", job="${job}"})) * 100`, swapUsedPct: `((node_memory_SwapTotal_bytes{instance="${node}",job="${job}"} - node_memory_SwapFree_bytes{instance="${node}",job="${job}"}) / (node_memory_SwapTotal_bytes{instance="${node}",job="${job}"})) * 100`, rootFsUsedPct: `100 - ((node_filesystem_avail_bytes{instance="${node}",job="${job}",mountpoint="/",fstype!="rootfs"} * 100) / node_filesystem_size_bytes{instance="${node}",job="${job}",mountpoint="/",fstype!="rootfs"})`, cpuCores: `count(count(node_cpu_seconds_total{instance="${node}",job="${job}"}) by (cpu))`, memTotal: `node_memory_MemTotal_bytes{instance="${node}",job="${job}"}`, uptime: `node_time_seconds{instance="${node}",job="${job}"} - node_boot_time_seconds{instance="${node}",job="${job}"}`, netRx: `sum(rate(node_network_receive_bytes_total{instance="${node}",job="${job}",device!~'tap.*|veth.*|br.*|docker.*|virbr*|podman.*|lo.*|vmbr.*|fwbr.|ip.*|gre.*|virbr.*|vnet.*'}[1m]))`, netTx: `sum(rate(node_network_transmit_bytes_total{instance="${node}",job="${job}",device!~'tap.*|veth.*|br.*|docker.*|virbr*|podman.*|lo.*|vmbr.*|fwbr.|ip.*|gre.*|virbr.*|vnet.*'}[1m]))`, sockstatTcp: `node_sockstat_TCP_inuse{instance="${node}",job="${job}"}`, sockstatTcpMem: `node_sockstat_TCP_mem{instance="${node}",job="${job}"} * 4096` // Converting pages to bytes (assuming 4KB pages) }; const results = {}; const queryPromises = Object.entries(queries).map(async ([key, expr]) => { try { const res = await query(url, expr); results[key] = res.length > 0 ? parseFloat(res[0].value[1]) : 0; } catch (e) { console.error(`[Prometheus] Error querying ${key} for ${node}:`, e.message); results[key] = 0; } }); await Promise.all(queryPromises); return results; } /** * Get historical metrics for a specific server (node) */ async function getServerHistory(baseUrl, instance, job, metric, range = '1h', start = null, end = null) { const url = normalizeUrl(baseUrl); const node = resolveToken(instance); // Custom multi-metric handler for CPU Busy if (metric === 'cpuBusy') { const modes = { system: 'system', user: 'user', iowait: 'iowait', irq: 'irq|softirq', other: 'nice|steal|guest|guest_nice', idle: 'idle' }; const rangeObj = parseRange(range, start, end); const timestamps = []; const series = {}; Object.keys(modes).forEach(m => series[m] = []); const results = await Promise.all(Object.entries(modes).map(async ([name, mode]) => { const expr = `avg(rate(node_cpu_seconds_total{mode=~"${mode}", instance="${node}"}[1m])) * 100`; const res = await queryRange(url, expr, rangeObj.queryStart, rangeObj.queryEnd, rangeObj.step); return { name, values: res.length > 0 ? res[0].values : [] }; })); if (results[0].values.length === 0) return { timestamps: [], series: {} }; // Use first result for timestamps results[0].values.forEach(v => timestamps.push(v[0] * 1000)); results.forEach(r => { r.values.forEach(v => series[r.name].push(parseFloat(v[1]))); }); return { timestamps, series }; } // Map metric keys to Prometheus expressions const metricMap = { sysLoad: `node_load1{instance="${node}",job="${job}"} * 100 / count(count(node_cpu_seconds_total{instance="${node}",job="${job}"}) by (cpu))`, memUsedPct: `(1 - (node_memory_MemAvailable_bytes{instance="${node}", job="${job}"} / node_memory_MemTotal_bytes{instance="${node}", job="${job}"})) * 100`, swapUsedPct: `((node_memory_SwapTotal_bytes{instance="${node}",job="${job}"} - node_memory_SwapFree_bytes{instance="${node}",job="${job}"}) / (node_memory_SwapTotal_bytes{instance="${node}",job="${job}"})) * 100`, rootFsUsedPct: `100 - ((node_filesystem_avail_bytes{instance="${node}",job="${job}",mountpoint="/",fstype!="rootfs"} * 100) / node_filesystem_size_bytes{instance="${node}",job="${job}",mountpoint="/",fstype!="rootfs"})`, netRx: `sum(rate(node_network_receive_bytes_total{instance="${node}",job="${job}",device!~'tap.*|veth.*|br.*|docker.*|virbr*|podman.*|lo.*|vmbr.*|fwbr.|ip.*|gre.*|virbr.*|vnet.*'}[1m]))`, netTx: `sum(rate(node_network_transmit_bytes_total{instance="${node}",job="${job}",device!~'tap.*|veth.*|br.*|docker.*|virbr*|podman.*|lo.*|vmbr.*|fwbr.|ip.*|gre.*|virbr.*|vnet.*'}[1m]))`, sockstatTcp: `node_sockstat_TCP_inuse{instance="${node}",job="${job}"}`, sockstatTcpMem: `node_sockstat_TCP_mem{instance="${node}",job="${job}"} * 4096` }; const expr = metricMap[metric]; if (!expr) throw new Error('Invalid metric for history'); const rangeObj = parseRange(range, start, end); try { const result = await queryRange(url, expr, rangeObj.queryStart, rangeObj.queryEnd, rangeObj.step); if (!result || result.length === 0) return { timestamps: [], values: [] }; return { timestamps: result[0].values.map(v => v[0] * 1000), values: result[0].values.map(v => parseFloat(v[1])) }; } catch (err) { console.error(`[Prometheus] Error fetching history for ${metric} on ${node}:`, err.message); throw err; } } function parseRange(range, start, end) { let duration, step, queryStart, queryEnd; if (start && end) { queryStart = Math.floor(new Date(start).getTime() / 1000); queryEnd = Math.floor(new Date(end).getTime() / 1000); duration = queryEnd - queryStart; step = Math.max(15, Math.floor(duration / 100)); } else { const rangeMap = { '15m': { duration: 900, step: 15 }, '30m': { duration: 1800, step: 30 }, '1h': { duration: 3600, step: 60 }, '6h': { duration: 21600, step: 300 }, '12h': { duration: 43200, step: 600 }, '24h': { duration: 86400, step: 900 }, '2d': { duration: 172800, step: 1800 }, '7d': { duration: 604800, step: 3600 } }; if (rangeMap[range]) { duration = rangeMap[range].duration; step = rangeMap[range].step; } else { const match = range.match(/^(\d+)([smhd])$/); if (match) { const val = parseInt(match[1]); const unit = match[2]; const multipliers = { s: 1, m: 60, h: 3600, d: 86400 }; duration = val * (multipliers[unit] || 3600); step = Math.max(15, Math.floor(duration / 100)); } else { duration = 3600; step = 60; } } queryEnd = Math.floor(Date.now() / 1000); queryStart = queryEnd - duration; } return { queryStart, queryEnd, step }; } module.exports = { testConnection, query, queryRange, getTargets, getOverviewMetrics, getNetworkHistory, mergeNetworkHistories, getCpuHistory, mergeCpuHistories, getServerDetails, getServerHistory };