feat: add notifications system with alert engine, email, scheduler, and traffic services
This commit is contained in:
@@ -12,6 +12,7 @@
|
||||
"dotenv": "^16.4.0",
|
||||
"express": "^4.21.0",
|
||||
"jsonwebtoken": "^9.0.3",
|
||||
"node-cron": "^3.0.0",
|
||||
"nodemailer": "^6.9.0",
|
||||
"ssh2": "^1.16.0"
|
||||
},
|
||||
@@ -19,6 +20,7 @@
|
||||
"@types/cors": "^2.8.17",
|
||||
"@types/express": "^5.0.0",
|
||||
"@types/jsonwebtoken": "^9.0.10",
|
||||
"@types/node-cron": "^3.0.0",
|
||||
"@types/nodemailer": "^6.4.0",
|
||||
"@types/ssh2": "^1.15.0",
|
||||
"tsx": "^4.19.0",
|
||||
|
||||
@@ -25,4 +25,12 @@ export const config = {
|
||||
],
|
||||
jwtSecret: process.env.JWT_SECRET ?? 'eventify-server-monitor-secret-key-change-in-production',
|
||||
},
|
||||
} as const;
|
||||
smtp: {
|
||||
host: process.env.SMTP_HOST ?? '',
|
||||
port: parseInt(process.env.SMTP_PORT ?? '587', 10),
|
||||
user: process.env.SMTP_USER ?? '',
|
||||
pass: process.env.SMTP_PASS ?? '',
|
||||
},
|
||||
dataDir: process.env.DATA_DIR ?? './data',
|
||||
appUrl: process.env.APP_URL ?? 'https://status.eventifyplus.com',
|
||||
};
|
||||
|
||||
@@ -2,18 +2,21 @@ import express from 'express';
|
||||
import cors from 'cors';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { dirname, resolve } from 'path';
|
||||
import { existsSync } from 'fs';
|
||||
import { existsSync, mkdirSync } from 'fs';
|
||||
import { config } from './config.js';
|
||||
import { healthRouter } from './routes/health.js';
|
||||
import { authRouter } from './routes/auth.js';
|
||||
import { usersRouter } from './routes/users.js';
|
||||
import { notificationRouter } from './routes/notifications.js';
|
||||
import { requireAuth } from './middleware/auth.js';
|
||||
import { sshManager } from './ssh/client.js';
|
||||
import { createNotification } from './services/notifications.js';
|
||||
import { startScheduler, stopScheduler } from './services/scheduler.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// Ensure data directory exists
|
||||
mkdirSync(config.dataDir, { recursive: true });
|
||||
|
||||
const app = express();
|
||||
|
||||
// ── Middleware ──
|
||||
@@ -27,8 +30,8 @@ app.use(express.json());
|
||||
|
||||
// ── API Routes ──
|
||||
app.use('/api/auth', authRouter);
|
||||
app.use('/api/notifications', requireAuth, notificationRouter);
|
||||
app.use('/api', requireAuth, healthRouter);
|
||||
app.use('/api', requireAuth, usersRouter);
|
||||
|
||||
// ── Static file serving for production SPA ──
|
||||
const clientDistPath = resolve(__dirname, '../../client/dist');
|
||||
@@ -45,101 +48,15 @@ if (existsSync(clientDistPath)) {
|
||||
const server = app.listen(config.port, () => {
|
||||
console.log(`[Server] Listening on http://localhost:${config.port}`);
|
||||
console.log(`[Server] SSH target: ${config.ssh.user}@${config.ssh.host}`);
|
||||
startScheduler();
|
||||
});
|
||||
|
||||
// ── Error monitoring: check health every 60s and alert on issues ──
|
||||
const MONITOR_INTERVAL = 60_000;
|
||||
let previousErrors: Set<string> = new Set();
|
||||
|
||||
async function monitorHealth() {
|
||||
try {
|
||||
const { getSystemInfo } = await import('./services/system.js');
|
||||
const { getDockerInfo } = await import('./services/docker.js');
|
||||
const { getMemoryInfo } = await import('./services/memory.js');
|
||||
const { getDiskInfo } = await import('./services/disk.js');
|
||||
|
||||
const currentErrors: Set<string> = new Set();
|
||||
|
||||
// Check system
|
||||
try {
|
||||
const sys = await getSystemInfo();
|
||||
if (sys.cpuPercent > 90) {
|
||||
const key = 'cpu-high';
|
||||
currentErrors.add(key);
|
||||
if (!previousErrors.has(key)) {
|
||||
await createNotification('warning', 'High CPU Usage', `CPU at ${sys.cpuPercent.toFixed(1)}%`, 'System Monitor');
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
const key = 'system-down';
|
||||
currentErrors.add(key);
|
||||
if (!previousErrors.has(key)) {
|
||||
await createNotification('error', 'System Check Failed', (e as Error).message, 'System Monitor');
|
||||
}
|
||||
}
|
||||
|
||||
// Check memory
|
||||
try {
|
||||
const mem = await getMemoryInfo();
|
||||
if (mem.usedPercent > 90) {
|
||||
const key = 'memory-high';
|
||||
currentErrors.add(key);
|
||||
if (!previousErrors.has(key)) {
|
||||
await createNotification('warning', 'High Memory Usage', `Memory at ${mem.usedPercent.toFixed(1)}%`, 'System Monitor');
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
|
||||
// Check disk
|
||||
try {
|
||||
const disk = await getDiskInfo();
|
||||
if (disk.usedPercent > 85) {
|
||||
const key = 'disk-high';
|
||||
currentErrors.add(key);
|
||||
if (!previousErrors.has(key)) {
|
||||
await createNotification('warning', 'Disk Space Low', `Disk at ${disk.usedPercent.toFixed(1)}% used`, 'System Monitor');
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
|
||||
// Check Docker containers
|
||||
try {
|
||||
const containers = await getDockerInfo();
|
||||
for (const c of containers) {
|
||||
if (c.status && !c.status.toLowerCase().includes('up')) {
|
||||
const key = `container-down-${c.name}`;
|
||||
currentErrors.add(key);
|
||||
if (!previousErrors.has(key)) {
|
||||
await createNotification('error', `Container Down: ${c.name}`, `Status: ${c.status}`, 'Docker Monitor');
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
|
||||
// Check for recoveries
|
||||
for (const prev of previousErrors) {
|
||||
if (!currentErrors.has(prev)) {
|
||||
await createNotification('recovery', `Recovered: ${prev}`, `The issue "${prev}" has been resolved.`, 'System Monitor');
|
||||
}
|
||||
}
|
||||
|
||||
previousErrors = currentErrors;
|
||||
} catch (err) {
|
||||
console.error('[Monitor] Health check error:', err);
|
||||
}
|
||||
}
|
||||
|
||||
// Start monitoring after SSH connection is ready (delay 10s)
|
||||
setTimeout(() => {
|
||||
monitorHealth();
|
||||
setInterval(monitorHealth, MONITOR_INTERVAL);
|
||||
console.log('[Monitor] Health monitoring started (60s interval)');
|
||||
}, 10_000);
|
||||
|
||||
// ── Graceful shutdown ──
|
||||
async function shutdown(signal: string) {
|
||||
console.log(`\n[Server] ${signal} received, shutting down gracefully...`);
|
||||
|
||||
stopScheduler();
|
||||
|
||||
server.close(() => {
|
||||
console.log('[Server] HTTP server closed');
|
||||
});
|
||||
|
||||
@@ -4,6 +4,7 @@ import { getMemoryInfo } from '../services/memory.js';
|
||||
import { getDiskInfo } from '../services/disk.js';
|
||||
import { getDockerInfo } from '../services/docker.js';
|
||||
import { getNginxStatus } from '../services/nginx.js';
|
||||
import { getTrafficInfo } from '../services/traffic.js';
|
||||
import type { OverviewResponse } from '../types/index.js';
|
||||
|
||||
export const healthRouter = Router();
|
||||
@@ -27,6 +28,7 @@ healthRouter.get('/overview', async (_req, res) => {
|
||||
const disk = await safe('disk', getDiskInfo);
|
||||
const docker = await safe('docker', getDockerInfo);
|
||||
const nginx = await safe('nginx', getNginxStatus);
|
||||
const traffic = await safe('traffic', getTrafficInfo);
|
||||
|
||||
const response: OverviewResponse = {
|
||||
system,
|
||||
@@ -34,6 +36,7 @@ healthRouter.get('/overview', async (_req, res) => {
|
||||
disk,
|
||||
docker,
|
||||
nginx,
|
||||
traffic,
|
||||
errors,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
133
server/src/routes/notifications.ts
Normal file
133
server/src/routes/notifications.ts
Normal file
@@ -0,0 +1,133 @@
|
||||
import { Router } from 'express';
|
||||
import {
|
||||
getAlerts,
|
||||
getThresholds,
|
||||
saveThresholds,
|
||||
getPreferences,
|
||||
savePreferences,
|
||||
updateAlert,
|
||||
deleteAlert,
|
||||
clearResolvedAlerts,
|
||||
} from '../services/store.js';
|
||||
import { sendTestEmail, sendSampleAlerts } from '../services/email.js';
|
||||
import type { ThresholdsConfig, NotificationPreferences } from '../types/index.js';
|
||||
|
||||
export const notificationRouter = Router();
|
||||
|
||||
// GET /api/notifications/summary
|
||||
notificationRouter.get('/summary', (_req, res) => {
|
||||
const alerts = getAlerts();
|
||||
const active = alerts.filter(a => a.resolvedAt === null);
|
||||
res.json({
|
||||
activeWarnings: active.filter(a => a.severity === 'warning').length,
|
||||
activeCriticals: active.filter(a => a.severity === 'critical').length,
|
||||
unacknowledged: alerts.filter(a => !a.acknowledged).length,
|
||||
total: alerts.length,
|
||||
});
|
||||
});
|
||||
|
||||
// GET /api/notifications/thresholds
|
||||
notificationRouter.get('/thresholds', (_req, res) => {
|
||||
res.json(getThresholds());
|
||||
});
|
||||
|
||||
// PUT /api/notifications/thresholds
|
||||
notificationRouter.put('/thresholds', (req, res) => {
|
||||
const body = req.body as ThresholdsConfig;
|
||||
if (!body?.thresholds || !Array.isArray(body.thresholds)) {
|
||||
res.status(400).json({ error: 'Invalid thresholds payload' });
|
||||
return;
|
||||
}
|
||||
saveThresholds(body);
|
||||
res.json(getThresholds());
|
||||
});
|
||||
|
||||
// GET /api/notifications/preferences
|
||||
notificationRouter.get('/preferences', (_req, res) => {
|
||||
res.json(getPreferences());
|
||||
});
|
||||
|
||||
// PUT /api/notifications/preferences
|
||||
notificationRouter.put('/preferences', (req, res) => {
|
||||
const body = req.body as NotificationPreferences;
|
||||
if (!body?.recipients || !Array.isArray(body.recipients)) {
|
||||
res.status(400).json({ error: 'Invalid preferences payload' });
|
||||
return;
|
||||
}
|
||||
savePreferences(body);
|
||||
res.json(getPreferences());
|
||||
});
|
||||
|
||||
// POST /api/notifications/test-email
|
||||
notificationRouter.post('/test-email', async (req, res) => {
|
||||
const { email } = req.body as { email?: string };
|
||||
if (!email) {
|
||||
res.status(400).json({ error: 'email is required' });
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await sendTestEmail(email);
|
||||
res.json({ success: true, message: `Test email sent to ${email}` });
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: (err as Error).message });
|
||||
}
|
||||
});
|
||||
|
||||
// POST /api/notifications/test-all — send sample emails for every alert type
|
||||
notificationRouter.post('/test-all', async (req, res) => {
|
||||
const { email } = req.body as { email?: string };
|
||||
if (!email) {
|
||||
res.status(400).json({ error: 'email is required' });
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const result = await sendSampleAlerts(email);
|
||||
res.json({ success: true, message: `Sent ${result.sent.length} sample emails to ${email}`, types: result.sent });
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: (err as Error).message });
|
||||
}
|
||||
});
|
||||
|
||||
// DELETE /api/notifications/clear
|
||||
notificationRouter.delete('/clear', (_req, res) => {
|
||||
const count = clearResolvedAlerts();
|
||||
res.json({ success: true, cleared: count });
|
||||
});
|
||||
|
||||
// GET /api/notifications — list with filters
|
||||
notificationRouter.get('/', (req, res) => {
|
||||
const page = Math.max(1, parseInt(String(req.query.page ?? '1'), 10));
|
||||
const limit = Math.min(100, Math.max(1, parseInt(String(req.query.limit ?? '20'), 10)));
|
||||
const status = String(req.query.status ?? 'all');
|
||||
|
||||
let alerts = getAlerts();
|
||||
|
||||
if (status === 'active') alerts = alerts.filter(a => a.resolvedAt === null);
|
||||
if (status === 'resolved') alerts = alerts.filter(a => a.resolvedAt !== null);
|
||||
|
||||
const total = alerts.length;
|
||||
const start = (page - 1) * limit;
|
||||
const items = alerts.slice(start, start + limit);
|
||||
|
||||
res.json({ alerts: items, total, page, limit });
|
||||
});
|
||||
|
||||
// PATCH /api/notifications/:id/acknowledge
|
||||
notificationRouter.patch('/:id/acknowledge', (req, res) => {
|
||||
const updated = updateAlert(req.params.id, { acknowledged: true });
|
||||
if (!updated) {
|
||||
res.status(404).json({ error: 'Alert not found' });
|
||||
return;
|
||||
}
|
||||
res.json(updated);
|
||||
});
|
||||
|
||||
// DELETE /api/notifications/:id
|
||||
notificationRouter.delete('/:id', (req, res) => {
|
||||
const ok = deleteAlert(req.params.id);
|
||||
if (!ok) {
|
||||
res.status(404).json({ error: 'Alert not found' });
|
||||
return;
|
||||
}
|
||||
res.json({ success: true });
|
||||
});
|
||||
245
server/src/services/alertEngine.ts
Normal file
245
server/src/services/alertEngine.ts
Normal file
@@ -0,0 +1,245 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
import { getThresholds, getPreferences, addAlert, getActiveAlerts, updateAlert } from './store.js';
|
||||
import { sendAlertEmail, sendRecoveryEmail } from './email.js';
|
||||
import type { OverviewResponse } from '../types/index.js';
|
||||
import type { AlertRecord, AlertSeverity, MetricKey } from '../types/notifications.js';
|
||||
|
||||
// In-memory cooldown map: "metric:severity" → last fired timestamp (ms)
|
||||
const cooldownMap = new Map<string, number>();
|
||||
|
||||
function isCooledDown(metric: MetricKey, severity: AlertSeverity, cooldownMs: number): boolean {
|
||||
const key = `${metric}:${severity}`;
|
||||
const last = cooldownMap.get(key);
|
||||
if (!last) return true;
|
||||
return Date.now() - last >= cooldownMs;
|
||||
}
|
||||
|
||||
function stampCooldown(metric: MetricKey, severity: AlertSeverity): void {
|
||||
cooldownMap.set(`${metric}:${severity}`, Date.now());
|
||||
}
|
||||
|
||||
function createAlert(
|
||||
metric: MetricKey,
|
||||
severity: AlertSeverity,
|
||||
message: string,
|
||||
value: number | string,
|
||||
threshold: number | string
|
||||
): AlertRecord {
|
||||
return {
|
||||
id: randomUUID(),
|
||||
metric,
|
||||
severity,
|
||||
message,
|
||||
value,
|
||||
threshold,
|
||||
firedAt: new Date().toISOString(),
|
||||
resolvedAt: null,
|
||||
emailSent: false,
|
||||
acknowledged: false,
|
||||
};
|
||||
}
|
||||
|
||||
export async function evaluateMetrics(overview: OverviewResponse): Promise<void> {
|
||||
const { thresholds } = getThresholds();
|
||||
const prefs = getPreferences();
|
||||
const cooldownMs = prefs.cooldownMinutes * 60_000;
|
||||
const newAlerts: AlertRecord[] = [];
|
||||
|
||||
for (const t of thresholds) {
|
||||
if (!t.enabled) continue;
|
||||
|
||||
// ── Extract metric value ──
|
||||
let value: number | string | null = null;
|
||||
|
||||
switch (t.metric) {
|
||||
case 'cpu_percent':
|
||||
value = overview.system?.cpuPercent ?? null;
|
||||
break;
|
||||
case 'load_avg_1m':
|
||||
value = overview.system?.loadAvg?.[0] ?? null;
|
||||
break;
|
||||
case 'memory_percent':
|
||||
value = overview.memory?.usedPercent ?? null;
|
||||
break;
|
||||
case 'disk_percent':
|
||||
value = overview.disk?.usedPercent ?? null;
|
||||
break;
|
||||
case 'nginx_status':
|
||||
value = overview.nginx?.status !== 'active' ? 1 : 0;
|
||||
break;
|
||||
case 'container_stopped': {
|
||||
const stopped = (overview.docker ?? []).filter(
|
||||
c => !c.status.toLowerCase().includes('up')
|
||||
).length;
|
||||
value = stopped;
|
||||
break;
|
||||
}
|
||||
case 'connections_total':
|
||||
value = overview.traffic?.totalConnections ?? null;
|
||||
break;
|
||||
case 'connections_per_ip':
|
||||
value = overview.traffic?.topIpConnections ?? null;
|
||||
break;
|
||||
case 'request_rate':
|
||||
value = overview.traffic?.requestsPerMinute ?? null;
|
||||
break;
|
||||
case 'error_rate_5xx':
|
||||
value = overview.traffic?.errors5xxPerMinute ?? null;
|
||||
break;
|
||||
}
|
||||
|
||||
// Metric unreachable
|
||||
if (value === null) {
|
||||
if (isCooledDown(t.metric, 'info', cooldownMs)) {
|
||||
const alert = createAlert(
|
||||
t.metric, 'info',
|
||||
`Metric "${t.metric}" could not be fetched`,
|
||||
'N/A', 'N/A'
|
||||
);
|
||||
addAlert(alert);
|
||||
newAlerts.push(alert);
|
||||
stampCooldown(t.metric, 'info');
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// ── Numeric threshold checks ──
|
||||
if (t.metric === 'nginx_status') {
|
||||
// Boolean: fires if nginx is not active
|
||||
if (value === 1) {
|
||||
const nginxStatus = overview.nginx?.status ?? 'unknown';
|
||||
if (isCooledDown(t.metric, 'critical', cooldownMs)) {
|
||||
// Resolve any existing warning first
|
||||
resolveMetricAlerts(t.metric, 'warning', newAlerts);
|
||||
const alert = createAlert(
|
||||
t.metric, 'critical',
|
||||
`Nginx is ${nginxStatus}`,
|
||||
nginxStatus, 'active'
|
||||
);
|
||||
addAlert(alert);
|
||||
newAlerts.push(alert);
|
||||
stampCooldown(t.metric, 'critical');
|
||||
}
|
||||
} else {
|
||||
resolveRecoveredAlerts(t.metric, newAlerts, prefs.recipients);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (t.metric === 'container_stopped') {
|
||||
if ((value as number) > 0) {
|
||||
const names = (overview.docker ?? [])
|
||||
.filter(c => !c.status.toLowerCase().includes('up'))
|
||||
.map(c => c.name)
|
||||
.join(', ');
|
||||
if (isCooledDown(t.metric, 'warning', cooldownMs)) {
|
||||
const alert = createAlert(
|
||||
t.metric, 'warning',
|
||||
`${value} container(s) not running: ${names}`,
|
||||
`${value} stopped`, '0 stopped'
|
||||
);
|
||||
addAlert(alert);
|
||||
newAlerts.push(alert);
|
||||
stampCooldown(t.metric, 'warning');
|
||||
}
|
||||
} else {
|
||||
resolveRecoveredAlerts(t.metric, newAlerts, prefs.recipients);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const numVal = value as number;
|
||||
|
||||
// Build context-rich messages for traffic metrics
|
||||
function buildMessage(severity: 'warning' | 'critical'): string {
|
||||
const level = severity === 'critical' ? 'critically high' : 'high';
|
||||
switch (t.metric) {
|
||||
case 'connections_total':
|
||||
return `${numVal} active TCP connections — potential DDoS or traffic spike (${level})`;
|
||||
case 'connections_per_ip': {
|
||||
const ip = overview.traffic?.topIp ?? 'unknown';
|
||||
return `Single IP (${ip}) has ${numVal} connections — possible DDoS attack (${level})`;
|
||||
}
|
||||
case 'request_rate':
|
||||
return `${numVal} HTTP requests/min — abnormally high API call volume (${level})`;
|
||||
case 'error_rate_5xx':
|
||||
return `${numVal} server errors (5xx) in the last minute — service degradation (${level})`;
|
||||
default:
|
||||
return `${t.metric.replace(/_/g, ' ')} is ${level} at ${numVal.toFixed(1)}`;
|
||||
}
|
||||
}
|
||||
|
||||
if (numVal >= t.critical) {
|
||||
// Escalate: resolve active warning first
|
||||
resolveMetricAlerts(t.metric, 'warning', newAlerts);
|
||||
if (isCooledDown(t.metric, 'critical', cooldownMs)) {
|
||||
const alert = createAlert(
|
||||
t.metric, 'critical',
|
||||
buildMessage('critical'),
|
||||
parseFloat(numVal.toFixed(2)), t.critical
|
||||
);
|
||||
addAlert(alert);
|
||||
newAlerts.push(alert);
|
||||
stampCooldown(t.metric, 'critical');
|
||||
}
|
||||
} else if (numVal >= t.warning) {
|
||||
if (isCooledDown(t.metric, 'warning', cooldownMs)) {
|
||||
const alert = createAlert(
|
||||
t.metric, 'warning',
|
||||
buildMessage('warning'),
|
||||
parseFloat(numVal.toFixed(2)), t.warning
|
||||
);
|
||||
addAlert(alert);
|
||||
newAlerts.push(alert);
|
||||
stampCooldown(t.metric, 'warning');
|
||||
}
|
||||
} else {
|
||||
// Value back to normal — resolve any active alerts
|
||||
resolveRecoveredAlerts(t.metric, newAlerts, prefs.recipients);
|
||||
}
|
||||
}
|
||||
|
||||
// Send emails for new alerts
|
||||
for (const alert of newAlerts) {
|
||||
if (!prefs.enabledSeverities.includes(alert.severity)) continue;
|
||||
const enabledRecipients = prefs.recipients.filter(r => r.enabled);
|
||||
if (!enabledRecipients.length) continue;
|
||||
try {
|
||||
await sendAlertEmail(alert, enabledRecipients);
|
||||
updateAlert(alert.id, { emailSent: true });
|
||||
} catch (err) {
|
||||
console.error(`[AlertEngine] Failed to send email for alert ${alert.id}:`, (err as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
if (newAlerts.length > 0) {
|
||||
console.log(`[AlertEngine] ${newAlerts.length} new alert(s) fired.`);
|
||||
}
|
||||
}
|
||||
|
||||
function resolveMetricAlerts(
|
||||
metric: MetricKey,
|
||||
severity: AlertSeverity,
|
||||
_newAlerts: AlertRecord[]
|
||||
): void {
|
||||
const active = getActiveAlerts(metric).filter(a => a.severity === severity);
|
||||
for (const a of active) {
|
||||
updateAlert(a.id, { resolvedAt: new Date().toISOString() });
|
||||
}
|
||||
}
|
||||
|
||||
function resolveRecoveredAlerts(
|
||||
metric: MetricKey,
|
||||
_newAlerts: AlertRecord[],
|
||||
recipients: { email: string; name: string; enabled: boolean }[]
|
||||
): void {
|
||||
const active = getActiveAlerts(metric);
|
||||
for (const a of active) {
|
||||
const resolved = updateAlert(a.id, { resolvedAt: new Date().toISOString() });
|
||||
if (resolved) {
|
||||
sendRecoveryEmail(resolved, recipients.filter(r => r.enabled)).catch((err: Error) =>
|
||||
console.error(`[AlertEngine] Recovery email failed:`, err.message)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
633
server/src/services/email.ts
Normal file
633
server/src/services/email.ts
Normal file
@@ -0,0 +1,633 @@
|
||||
import nodemailer from 'nodemailer';
|
||||
import { config } from '../config.js';
|
||||
import type { AlertRecord, EmailRecipient } from '../types/index.js';
|
||||
|
||||
function createTransporter() {
|
||||
return nodemailer.createTransport({
|
||||
host: config.smtp.host,
|
||||
port: config.smtp.port,
|
||||
secure: false,
|
||||
auth: {
|
||||
user: config.smtp.user,
|
||||
pass: config.smtp.pass,
|
||||
},
|
||||
tls: { rejectUnauthorized: false },
|
||||
});
|
||||
}
|
||||
|
||||
const METRIC_LABELS: Record<string, string> = {
|
||||
cpu_percent: 'CPU Usage',
|
||||
load_avg_1m: 'Load Average (1m)',
|
||||
memory_percent: 'Memory Usage',
|
||||
disk_percent: 'Disk Usage',
|
||||
nginx_status: 'Nginx Status',
|
||||
container_stopped: 'Container Status',
|
||||
connections_total: 'Total Connections',
|
||||
connections_per_ip: 'Connections per IP',
|
||||
request_rate: 'Request Rate',
|
||||
error_rate_5xx: '5xx Error Rate',
|
||||
};
|
||||
|
||||
// ── Severity theme config ──
|
||||
interface SeverityTheme {
|
||||
gradient: string;
|
||||
badgeBg: string;
|
||||
badgeText: string;
|
||||
accentColor: string;
|
||||
icon: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
const SEVERITY_THEMES: Record<string, SeverityTheme> = {
|
||||
critical: {
|
||||
gradient: 'linear-gradient(135deg, #dc2626 0%, #991b1b 100%)',
|
||||
badgeBg: '#fef2f2',
|
||||
badgeText: '#dc2626',
|
||||
accentColor: '#dc2626',
|
||||
icon: '⚠', // ⚠
|
||||
label: 'CRITICAL',
|
||||
},
|
||||
warning: {
|
||||
gradient: 'linear-gradient(135deg, #f59e0b 0%, #d97706 100%)',
|
||||
badgeBg: '#fffbeb',
|
||||
badgeText: '#d97706',
|
||||
accentColor: '#d97706',
|
||||
icon: '⚠', // ⚠
|
||||
label: 'WARNING',
|
||||
},
|
||||
info: {
|
||||
gradient: 'linear-gradient(135deg, #3b82f6 0%, #1d4ed8 100%)',
|
||||
badgeBg: '#eff6ff',
|
||||
badgeText: '#2563eb',
|
||||
accentColor: '#2563eb',
|
||||
icon: 'ℹ', // ℹ
|
||||
label: 'INFO',
|
||||
},
|
||||
};
|
||||
|
||||
const RECOVERY_THEME = {
|
||||
gradient: 'linear-gradient(135deg, #22c55e 0%, #15803d 100%)',
|
||||
badgeBg: '#f0fdf4',
|
||||
badgeText: '#16a34a',
|
||||
accentColor: '#16a34a',
|
||||
icon: '✔', // ✔
|
||||
label: 'RESOLVED',
|
||||
};
|
||||
|
||||
// ── Shared layout wrapper ──
|
||||
function emailWrapper(content: string): string {
|
||||
return `<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta name="color-scheme" content="light">
|
||||
<!--[if mso]><noscript><xml><o:OfficeDocumentSettings><o:AllowPNG/><o:PixelsPerInch>96</o:PixelsPerInch></o:OfficeDocumentSettings></xml></noscript><![endif]-->
|
||||
</head>
|
||||
<body style="margin:0;padding:0;background-color:#f1f5f9;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,'Helvetica Neue',Arial,sans-serif;-webkit-font-smoothing:antialiased;">
|
||||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="background-color:#f1f5f9;">
|
||||
<tr>
|
||||
<td align="center" style="padding:40px 16px;">
|
||||
<table role="presentation" width="560" cellpadding="0" cellspacing="0" style="max-width:560px;width:100%;">
|
||||
${content}
|
||||
<!-- Footer -->
|
||||
<tr>
|
||||
<td style="padding:24px 0 0;">
|
||||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0">
|
||||
<tr>
|
||||
<td align="center" style="padding:16px 24px;border-top:1px solid #e2e8f0;">
|
||||
<table role="presentation" cellpadding="0" cellspacing="0">
|
||||
<tr>
|
||||
<td style="padding-right:8px;vertical-align:middle;">
|
||||
<div style="width:24px;height:24px;background:linear-gradient(135deg,#6366f1 0%,#4f46e5 100%);border-radius:6px;text-align:center;line-height:24px;color:#fff;font-size:13px;font-weight:700;">E</div>
|
||||
</td>
|
||||
<td style="vertical-align:middle;">
|
||||
<span style="font-size:13px;font-weight:600;color:#475569;">Eventify Server Monitor</span>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
<p style="margin:8px 0 0;font-size:12px;color:#94a3b8;">
|
||||
<a href="${config.appUrl}" style="color:#6366f1;text-decoration:none;">status.eventifyplus.com</a>
|
||||
· Automated alert — do not reply
|
||||
</p>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>`;
|
||||
}
|
||||
|
||||
// ── Alert email (critical / warning / info) ──
|
||||
function buildAlertHtml(alert: AlertRecord): string {
|
||||
const theme = SEVERITY_THEMES[alert.severity] ?? SEVERITY_THEMES.info;
|
||||
const label = METRIC_LABELS[alert.metric] ?? alert.metric;
|
||||
const firedAt = new Date(alert.firedAt).toLocaleString('en-IN', { timeZone: 'Asia/Kolkata' });
|
||||
|
||||
const content = `
|
||||
<!-- Header with gradient -->
|
||||
<tr>
|
||||
<td style="background:${theme.gradient};padding:32px 32px 28px;border-radius:12px 12px 0 0;">
|
||||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0">
|
||||
<tr>
|
||||
<td>
|
||||
<!-- Severity badge -->
|
||||
<div style="display:inline-block;background:rgba(255,255,255,0.2);border-radius:20px;padding:4px 14px 4px 10px;margin-bottom:16px;">
|
||||
<span style="color:#fff;font-size:12px;font-weight:700;letter-spacing:1.5px;">${theme.icon} ${theme.label} ALERT</span>
|
||||
</div>
|
||||
<h1 style="margin:0 0 6px;color:#ffffff;font-size:22px;font-weight:700;line-height:1.3;">
|
||||
${label} Threshold Breached
|
||||
</h1>
|
||||
<p style="margin:0;color:rgba(255,255,255,0.85);font-size:14px;">
|
||||
Detected at ${firedAt} IST
|
||||
</p>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<!-- Body card -->
|
||||
<tr>
|
||||
<td style="background:#ffffff;padding:0;border-left:1px solid #e2e8f0;border-right:1px solid #e2e8f0;">
|
||||
<!-- Metric value highlight -->
|
||||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0">
|
||||
<tr>
|
||||
<td style="padding:28px 32px 20px;">
|
||||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="background:${theme.badgeBg};border-radius:10px;border:1px solid ${theme.accentColor}20;">
|
||||
<tr>
|
||||
<td style="padding:20px 24px;">
|
||||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0">
|
||||
<tr>
|
||||
<td width="50%">
|
||||
<p style="margin:0 0 4px;font-size:11px;text-transform:uppercase;letter-spacing:1px;color:#64748b;font-weight:600;">Current Value</p>
|
||||
<p style="margin:0;font-size:28px;font-weight:800;color:${theme.accentColor};line-height:1.2;">${alert.value}</p>
|
||||
</td>
|
||||
<td width="50%" style="text-align:right;">
|
||||
<p style="margin:0 0 4px;font-size:11px;text-transform:uppercase;letter-spacing:1px;color:#64748b;font-weight:600;">Threshold</p>
|
||||
<p style="margin:0;font-size:28px;font-weight:800;color:#334155;line-height:1.2;">${alert.threshold}</p>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<!-- Details -->
|
||||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0">
|
||||
<tr>
|
||||
<td style="padding:0 32px 28px;">
|
||||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="border-collapse:separate;border-spacing:0;">
|
||||
<!-- Metric -->
|
||||
<tr>
|
||||
<td style="padding:12px 0;border-bottom:1px solid #f1f5f9;" width="140">
|
||||
<span style="font-size:13px;color:#64748b;font-weight:500;">Metric</span>
|
||||
</td>
|
||||
<td style="padding:12px 0;border-bottom:1px solid #f1f5f9;">
|
||||
<span style="font-size:14px;color:#0f172a;font-weight:600;">${label}</span>
|
||||
</td>
|
||||
</tr>
|
||||
<!-- Severity -->
|
||||
<tr>
|
||||
<td style="padding:12px 0;border-bottom:1px solid #f1f5f9;">
|
||||
<span style="font-size:13px;color:#64748b;font-weight:500;">Severity</span>
|
||||
</td>
|
||||
<td style="padding:12px 0;border-bottom:1px solid #f1f5f9;">
|
||||
<span style="display:inline-block;background:${theme.badgeBg};color:${theme.badgeText};font-size:12px;font-weight:700;padding:3px 10px;border-radius:12px;border:1px solid ${theme.accentColor}30;text-transform:uppercase;letter-spacing:0.5px;">${alert.severity}</span>
|
||||
</td>
|
||||
</tr>
|
||||
<!-- Message -->
|
||||
<tr>
|
||||
<td style="padding:12px 0;" valign="top">
|
||||
<span style="font-size:13px;color:#64748b;font-weight:500;">Details</span>
|
||||
</td>
|
||||
<td style="padding:12px 0;">
|
||||
<span style="font-size:14px;color:#334155;line-height:1.5;">${alert.message}</span>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<!-- CTA Button -->
|
||||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0">
|
||||
<tr>
|
||||
<td style="padding:0 32px 32px;">
|
||||
<table role="presentation" cellpadding="0" cellspacing="0">
|
||||
<tr>
|
||||
<td style="background:${theme.gradient};border-radius:8px;">
|
||||
<a href="${config.appUrl}" target="_blank" style="display:inline-block;padding:12px 28px;color:#ffffff;text-decoration:none;font-size:14px;font-weight:600;letter-spacing:0.3px;">
|
||||
View Dashboard →
|
||||
</a>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<!-- Bottom border radius -->
|
||||
<tr>
|
||||
<td style="background:#ffffff;height:4px;border-radius:0 0 12px 12px;border-left:1px solid #e2e8f0;border-right:1px solid #e2e8f0;border-bottom:1px solid #e2e8f0;"></td>
|
||||
</tr>`;
|
||||
|
||||
return emailWrapper(content);
|
||||
}
|
||||
|
||||
// ── Recovery email ──
|
||||
function buildRecoveryHtml(alert: AlertRecord): string {
|
||||
const theme = RECOVERY_THEME;
|
||||
const label = METRIC_LABELS[alert.metric] ?? alert.metric;
|
||||
const firedAt = new Date(alert.firedAt).toLocaleString('en-IN', { timeZone: 'Asia/Kolkata' });
|
||||
const resolvedAt = alert.resolvedAt
|
||||
? new Date(alert.resolvedAt).toLocaleString('en-IN', { timeZone: 'Asia/Kolkata' })
|
||||
: 'N/A';
|
||||
|
||||
const content = `
|
||||
<!-- Header with gradient -->
|
||||
<tr>
|
||||
<td style="background:${theme.gradient};padding:32px 32px 28px;border-radius:12px 12px 0 0;">
|
||||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0">
|
||||
<tr>
|
||||
<td>
|
||||
<div style="display:inline-block;background:rgba(255,255,255,0.2);border-radius:20px;padding:4px 14px 4px 10px;margin-bottom:16px;">
|
||||
<span style="color:#fff;font-size:12px;font-weight:700;letter-spacing:1.5px;">${theme.icon} ${theme.label}</span>
|
||||
</div>
|
||||
<h1 style="margin:0 0 6px;color:#ffffff;font-size:22px;font-weight:700;line-height:1.3;">
|
||||
${label} — Back to Normal
|
||||
</h1>
|
||||
<p style="margin:0;color:rgba(255,255,255,0.85);font-size:14px;">
|
||||
Resolved at ${resolvedAt} IST
|
||||
</p>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<!-- Body card -->
|
||||
<tr>
|
||||
<td style="background:#ffffff;padding:0;border-left:1px solid #e2e8f0;border-right:1px solid #e2e8f0;">
|
||||
<!-- Recovery status highlight -->
|
||||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0">
|
||||
<tr>
|
||||
<td style="padding:28px 32px 20px;">
|
||||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="background:${theme.badgeBg};border-radius:10px;border:1px solid ${theme.accentColor}20;">
|
||||
<tr>
|
||||
<td style="padding:20px 24px;text-align:center;">
|
||||
<div style="font-size:36px;line-height:1;">✓</div>
|
||||
<p style="margin:8px 0 0;font-size:18px;font-weight:700;color:${theme.accentColor};">All Clear</p>
|
||||
<p style="margin:4px 0 0;font-size:13px;color:#64748b;">The metric has returned to acceptable levels</p>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<!-- Timeline -->
|
||||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0">
|
||||
<tr>
|
||||
<td style="padding:0 32px 28px;">
|
||||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0">
|
||||
<tr>
|
||||
<td style="padding:12px 0;border-bottom:1px solid #f1f5f9;" width="140">
|
||||
<span style="font-size:13px;color:#64748b;font-weight:500;">Metric</span>
|
||||
</td>
|
||||
<td style="padding:12px 0;border-bottom:1px solid #f1f5f9;">
|
||||
<span style="font-size:14px;color:#0f172a;font-weight:600;">${label}</span>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:12px 0;border-bottom:1px solid #f1f5f9;">
|
||||
<span style="font-size:13px;color:#64748b;font-weight:500;">Alert Fired</span>
|
||||
</td>
|
||||
<td style="padding:12px 0;border-bottom:1px solid #f1f5f9;">
|
||||
<span style="font-size:14px;color:#334155;">${firedAt} IST</span>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:12px 0;border-bottom:1px solid #f1f5f9;">
|
||||
<span style="font-size:13px;color:#64748b;font-weight:500;">Resolved</span>
|
||||
</td>
|
||||
<td style="padding:12px 0;border-bottom:1px solid #f1f5f9;">
|
||||
<span style="font-size:14px;color:${theme.accentColor};font-weight:600;">${resolvedAt} IST</span>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="padding:12px 0;">
|
||||
<span style="font-size:13px;color:#64748b;font-weight:500;">Previous Alert</span>
|
||||
</td>
|
||||
<td style="padding:12px 0;">
|
||||
<span style="font-size:14px;color:#334155;">${alert.message}</span>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<!-- CTA Button -->
|
||||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0">
|
||||
<tr>
|
||||
<td style="padding:0 32px 32px;">
|
||||
<table role="presentation" cellpadding="0" cellspacing="0">
|
||||
<tr>
|
||||
<td style="background:${theme.gradient};border-radius:8px;">
|
||||
<a href="${config.appUrl}" target="_blank" style="display:inline-block;padding:12px 28px;color:#ffffff;text-decoration:none;font-size:14px;font-weight:600;letter-spacing:0.3px;">
|
||||
View Dashboard →
|
||||
</a>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<!-- Bottom border radius -->
|
||||
<tr>
|
||||
<td style="background:#ffffff;height:4px;border-radius:0 0 12px 12px;border-left:1px solid #e2e8f0;border-right:1px solid #e2e8f0;border-bottom:1px solid #e2e8f0;"></td>
|
||||
</tr>`;
|
||||
|
||||
return emailWrapper(content);
|
||||
}
|
||||
|
||||
// ── Test email (basic SMTP check) ──
|
||||
function buildTestHtml(): string {
|
||||
const content = `
|
||||
<!-- Header -->
|
||||
<tr>
|
||||
<td style="background:linear-gradient(135deg,#6366f1 0%,#4f46e5 100%);padding:32px 32px 28px;border-radius:12px 12px 0 0;">
|
||||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0">
|
||||
<tr>
|
||||
<td>
|
||||
<div style="display:inline-block;background:rgba(255,255,255,0.2);border-radius:20px;padding:4px 14px 4px 10px;margin-bottom:16px;">
|
||||
<span style="color:#fff;font-size:12px;font-weight:700;letter-spacing:1.5px;">⚡ SMTP TEST</span>
|
||||
</div>
|
||||
<h1 style="margin:0 0 6px;color:#ffffff;font-size:22px;font-weight:700;">
|
||||
Connection Verified
|
||||
</h1>
|
||||
<p style="margin:0;color:rgba(255,255,255,0.85);font-size:14px;">
|
||||
Your SMTP settings are configured correctly
|
||||
</p>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<!-- Body -->
|
||||
<tr>
|
||||
<td style="background:#ffffff;padding:28px 32px 32px;border-left:1px solid #e2e8f0;border-right:1px solid #e2e8f0;">
|
||||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="background:#f0fdf4;border-radius:10px;border:1px solid #bbf7d0;">
|
||||
<tr>
|
||||
<td style="padding:20px 24px;text-align:center;">
|
||||
<div style="font-size:36px;line-height:1;">✓</div>
|
||||
<p style="margin:8px 0 0;font-size:16px;font-weight:700;color:#16a34a;">SMTP is Working</p>
|
||||
<p style="margin:6px 0 0;font-size:13px;color:#64748b;line-height:1.5;">
|
||||
This confirms that Eventify Server Monitor can send<br>alert and recovery emails through your mail server.
|
||||
</p>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="margin-top:20px;">
|
||||
<tr>
|
||||
<td>
|
||||
<table role="presentation" cellpadding="0" cellspacing="0">
|
||||
<tr>
|
||||
<td style="background:linear-gradient(135deg,#6366f1 0%,#4f46e5 100%);border-radius:8px;">
|
||||
<a href="${config.appUrl}" target="_blank" style="display:inline-block;padding:12px 28px;color:#ffffff;text-decoration:none;font-size:14px;font-weight:600;letter-spacing:0.3px;">
|
||||
View Dashboard →
|
||||
</a>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
<!-- Bottom border radius -->
|
||||
<tr>
|
||||
<td style="background:#ffffff;height:4px;border-radius:0 0 12px 12px;border-left:1px solid #e2e8f0;border-right:1px solid #e2e8f0;border-bottom:1px solid #e2e8f0;"></td>
|
||||
</tr>`;
|
||||
|
||||
return emailWrapper(content);
|
||||
}
|
||||
|
||||
// ── Public send functions ──
|
||||
|
||||
export async function sendAlertEmail(alert: AlertRecord, recipients: EmailRecipient[]): Promise<void> {
|
||||
const enabled = recipients.filter(r => r.enabled);
|
||||
if (!enabled.length || !config.smtp.host) return;
|
||||
|
||||
const transporter = createTransporter();
|
||||
const label = METRIC_LABELS[alert.metric] ?? alert.metric;
|
||||
const subject = `[${alert.severity.toUpperCase()}] ${label} alert — Eventify Server`;
|
||||
|
||||
await transporter.sendMail({
|
||||
from: `"Eventify Monitor" <${config.smtp.user}>`,
|
||||
to: enabled.map(r => `"${r.name}" <${r.email}>`).join(', '),
|
||||
subject,
|
||||
html: buildAlertHtml(alert),
|
||||
});
|
||||
}
|
||||
|
||||
export async function sendRecoveryEmail(alert: AlertRecord, recipients: EmailRecipient[]): Promise<void> {
|
||||
const enabled = recipients.filter(r => r.enabled);
|
||||
if (!enabled.length || !config.smtp.host) return;
|
||||
|
||||
const transporter = createTransporter();
|
||||
const label = METRIC_LABELS[alert.metric] ?? alert.metric;
|
||||
|
||||
await transporter.sendMail({
|
||||
from: `"Eventify Monitor" <${config.smtp.user}>`,
|
||||
to: enabled.map(r => `"${r.name}" <${r.email}>`).join(', '),
|
||||
subject: `[RESOLVED] ${label} — Eventify Server`,
|
||||
html: buildRecoveryHtml(alert),
|
||||
});
|
||||
}
|
||||
|
||||
export async function sendTestEmail(to: string): Promise<void> {
|
||||
if (!config.smtp.host) throw new Error('SMTP not configured');
|
||||
const transporter = createTransporter();
|
||||
await transporter.sendMail({
|
||||
from: `"Eventify Monitor" <${config.smtp.user}>`,
|
||||
to,
|
||||
subject: 'SMTP Test — Eventify Server Monitor',
|
||||
html: buildTestHtml(),
|
||||
});
|
||||
}
|
||||
|
||||
// ── Send sample emails for every alert type ──
|
||||
export async function sendSampleAlerts(to: string): Promise<{ sent: string[] }> {
|
||||
if (!config.smtp.host) throw new Error('SMTP not configured');
|
||||
const transporter = createTransporter();
|
||||
const from = `"Eventify Monitor" <${config.smtp.user}>`;
|
||||
const now = new Date().toISOString();
|
||||
const sent: string[] = [];
|
||||
|
||||
// 1. Critical alert
|
||||
const criticalAlert: AlertRecord = {
|
||||
id: 'sample-critical',
|
||||
metric: 'cpu_percent',
|
||||
severity: 'critical',
|
||||
message: 'CPU utilization is critically high at 97.2% — sustained for over 5 minutes',
|
||||
value: 97.2,
|
||||
threshold: 95,
|
||||
firedAt: now,
|
||||
resolvedAt: null,
|
||||
emailSent: true,
|
||||
acknowledged: false,
|
||||
};
|
||||
await transporter.sendMail({
|
||||
from,
|
||||
to,
|
||||
subject: '[CRITICAL] CPU Usage alert — Eventify Server (Sample)',
|
||||
html: buildAlertHtml(criticalAlert),
|
||||
});
|
||||
sent.push('critical');
|
||||
|
||||
// 2. Warning alert
|
||||
const warningAlert: AlertRecord = {
|
||||
id: 'sample-warning',
|
||||
metric: 'memory_percent',
|
||||
severity: 'warning',
|
||||
message: 'Memory usage is high at 82.5% — consider checking for memory leaks',
|
||||
value: 82.5,
|
||||
threshold: 80,
|
||||
firedAt: now,
|
||||
resolvedAt: null,
|
||||
emailSent: true,
|
||||
acknowledged: false,
|
||||
};
|
||||
await transporter.sendMail({
|
||||
from,
|
||||
to,
|
||||
subject: '[WARNING] Memory Usage alert — Eventify Server (Sample)',
|
||||
html: buildAlertHtml(warningAlert),
|
||||
});
|
||||
sent.push('warning');
|
||||
|
||||
// 3. Info alert
|
||||
const infoAlert: AlertRecord = {
|
||||
id: 'sample-info',
|
||||
metric: 'nginx_status',
|
||||
severity: 'info',
|
||||
message: 'Metric "nginx_status" could not be fetched — service may be temporarily unreachable',
|
||||
value: 'N/A',
|
||||
threshold: 'N/A',
|
||||
firedAt: now,
|
||||
resolvedAt: null,
|
||||
emailSent: true,
|
||||
acknowledged: false,
|
||||
};
|
||||
await transporter.sendMail({
|
||||
from,
|
||||
to,
|
||||
subject: '[INFO] Nginx Status alert — Eventify Server (Sample)',
|
||||
html: buildAlertHtml(infoAlert),
|
||||
});
|
||||
sent.push('info');
|
||||
|
||||
// 4. DDoS / High connections alert (critical)
|
||||
const ddosAlert: AlertRecord = {
|
||||
id: 'sample-ddos',
|
||||
metric: 'connections_per_ip',
|
||||
severity: 'critical',
|
||||
message: 'Single IP (185.220.101.42) has 347 connections — possible DDoS attack (critically high)',
|
||||
value: 347,
|
||||
threshold: 100,
|
||||
firedAt: now,
|
||||
resolvedAt: null,
|
||||
emailSent: true,
|
||||
acknowledged: false,
|
||||
};
|
||||
await transporter.sendMail({
|
||||
from,
|
||||
to,
|
||||
subject: '[CRITICAL] Connections per IP alert — Eventify Server (Sample)',
|
||||
html: buildAlertHtml(ddosAlert),
|
||||
});
|
||||
sent.push('ddos');
|
||||
|
||||
// 5. High API request rate (warning)
|
||||
const apiRateAlert: AlertRecord = {
|
||||
id: 'sample-api-rate',
|
||||
metric: 'request_rate',
|
||||
severity: 'warning',
|
||||
message: '2,847 HTTP requests/min — abnormally high API call volume (high)',
|
||||
value: 2847,
|
||||
threshold: 1000,
|
||||
firedAt: now,
|
||||
resolvedAt: null,
|
||||
emailSent: true,
|
||||
acknowledged: false,
|
||||
};
|
||||
await transporter.sendMail({
|
||||
from,
|
||||
to,
|
||||
subject: '[WARNING] Request Rate alert — Eventify Server (Sample)',
|
||||
html: buildAlertHtml(apiRateAlert),
|
||||
});
|
||||
sent.push('api-rate');
|
||||
|
||||
// 6. 5xx error spike (critical)
|
||||
const errorAlert: AlertRecord = {
|
||||
id: 'sample-5xx',
|
||||
metric: 'error_rate_5xx',
|
||||
severity: 'critical',
|
||||
message: '73 server errors (5xx) in the last minute — service degradation (critically high)',
|
||||
value: 73,
|
||||
threshold: 50,
|
||||
firedAt: now,
|
||||
resolvedAt: null,
|
||||
emailSent: true,
|
||||
acknowledged: false,
|
||||
};
|
||||
await transporter.sendMail({
|
||||
from,
|
||||
to,
|
||||
subject: '[CRITICAL] 5xx Error Rate alert — Eventify Server (Sample)',
|
||||
html: buildAlertHtml(errorAlert),
|
||||
});
|
||||
sent.push('5xx-errors');
|
||||
|
||||
// 7. Recovery email
|
||||
const recoveredAlert: AlertRecord = {
|
||||
id: 'sample-recovery',
|
||||
metric: 'connections_total',
|
||||
severity: 'warning',
|
||||
message: 'Total connections were high at 1,247 — traffic has normalized',
|
||||
value: 1247,
|
||||
threshold: 500,
|
||||
firedAt: new Date(Date.now() - 45 * 60_000).toISOString(), // 45 min ago
|
||||
resolvedAt: now,
|
||||
emailSent: true,
|
||||
acknowledged: false,
|
||||
};
|
||||
await transporter.sendMail({
|
||||
from,
|
||||
to,
|
||||
subject: '[RESOLVED] Total Connections — Eventify Server (Sample)',
|
||||
html: buildRecoveryHtml(recoveredAlert),
|
||||
});
|
||||
sent.push('recovery');
|
||||
|
||||
return { sent };
|
||||
}
|
||||
66
server/src/services/scheduler.ts
Normal file
66
server/src/services/scheduler.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
import cron from 'node-cron';
|
||||
import { getSystemInfo } from './system.js';
|
||||
import { getMemoryInfo } from './memory.js';
|
||||
import { getDiskInfo } from './disk.js';
|
||||
import { getDockerInfo } from './docker.js';
|
||||
import { getNginxStatus } from './nginx.js';
|
||||
import { getTrafficInfo } from './traffic.js';
|
||||
import { evaluateMetrics } from './alertEngine.js';
|
||||
import type { OverviewResponse } from '../types/index.js';
|
||||
|
||||
let task: cron.ScheduledTask | null = null;
|
||||
|
||||
async function runHealthCheck(): Promise<void> {
|
||||
const errors: Record<string, string> = {};
|
||||
|
||||
const safe = async <T>(name: string, fn: () => Promise<T>): Promise<T | null> => {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (err) {
|
||||
errors[name] = (err as Error).message ?? 'Unknown error';
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
const system = await safe('system', getSystemInfo);
|
||||
const memory = await safe('memory', getMemoryInfo);
|
||||
const disk = await safe('disk', getDiskInfo);
|
||||
const docker = await safe('docker', getDockerInfo);
|
||||
const nginx = await safe('nginx', getNginxStatus);
|
||||
const traffic = await safe('traffic', getTrafficInfo);
|
||||
|
||||
const overview: OverviewResponse = {
|
||||
system,
|
||||
memory,
|
||||
disk,
|
||||
docker,
|
||||
nginx,
|
||||
traffic,
|
||||
errors,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
|
||||
try {
|
||||
await evaluateMetrics(overview);
|
||||
} catch (err) {
|
||||
console.error('[Scheduler] Alert evaluation error:', (err as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
export function startScheduler(): void {
|
||||
if (task) return;
|
||||
console.log('[Scheduler] Starting health check scheduler (every 1 minute)');
|
||||
task = cron.schedule('*/1 * * * *', () => {
|
||||
runHealthCheck().catch(err =>
|
||||
console.error('[Scheduler] Health check failed:', (err as Error).message)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export function stopScheduler(): void {
|
||||
if (task) {
|
||||
task.stop();
|
||||
task = null;
|
||||
console.log('[Scheduler] Stopped');
|
||||
}
|
||||
}
|
||||
143
server/src/services/store.ts
Normal file
143
server/src/services/store.ts
Normal file
@@ -0,0 +1,143 @@
|
||||
import { readFileSync, writeFileSync, renameSync, mkdirSync, existsSync } from 'fs';
|
||||
import { resolve } from 'path';
|
||||
import { config } from '../config.js';
|
||||
import type { AlertRecord, ThresholdsConfig, NotificationPreferences } from '../types/index.js';
|
||||
|
||||
const MAX_ALERTS = 500;
|
||||
|
||||
function dataPath(filename: string): string {
|
||||
return resolve(config.dataDir, filename);
|
||||
}
|
||||
|
||||
function ensureDir(): void {
|
||||
if (!existsSync(config.dataDir)) {
|
||||
mkdirSync(config.dataDir, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
function readJson<T>(filename: string, fallback: T): T {
|
||||
ensureDir();
|
||||
const path = dataPath(filename);
|
||||
if (!existsSync(path)) return fallback;
|
||||
try {
|
||||
return JSON.parse(readFileSync(path, 'utf8')) as T;
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
function writeJson<T>(filename: string, data: T): void {
|
||||
ensureDir();
|
||||
const path = dataPath(filename);
|
||||
const tmp = path + '.tmp';
|
||||
writeFileSync(tmp, JSON.stringify(data, null, 2), 'utf8');
|
||||
renameSync(tmp, path);
|
||||
}
|
||||
|
||||
// ── Thresholds ──
|
||||
|
||||
const defaultThresholds: ThresholdsConfig = {
|
||||
thresholds: [
|
||||
{ metric: 'cpu_percent', warning: 80, critical: 95, enabled: true },
|
||||
{ metric: 'load_avg_1m', warning: 4, critical: 8, enabled: true },
|
||||
{ metric: 'memory_percent', warning: 80, critical: 95, enabled: true },
|
||||
{ metric: 'disk_percent', warning: 80, critical: 95, enabled: true },
|
||||
{ metric: 'nginx_status', warning: 0, critical: 0, enabled: true },
|
||||
{ metric: 'container_stopped', warning: 0, critical: 0, enabled: true },
|
||||
{ metric: 'connections_total', warning: 500, critical: 1000, enabled: true },
|
||||
{ metric: 'connections_per_ip', warning: 50, critical: 100, enabled: true },
|
||||
{ metric: 'request_rate', warning: 1000, critical: 3000, enabled: true },
|
||||
{ metric: 'error_rate_5xx', warning: 10, critical: 50, enabled: true },
|
||||
],
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
export function getThresholds(): ThresholdsConfig {
|
||||
const saved = readJson<ThresholdsConfig>('thresholds.json', defaultThresholds);
|
||||
// Merge any new default metrics that don't exist in the saved file
|
||||
const savedMetrics = new Set(saved.thresholds.map(t => t.metric));
|
||||
const missing = defaultThresholds.thresholds.filter(t => !savedMetrics.has(t.metric));
|
||||
if (missing.length > 0) {
|
||||
saved.thresholds.push(...missing);
|
||||
writeJson('thresholds.json', saved);
|
||||
}
|
||||
return saved;
|
||||
}
|
||||
|
||||
export function saveThresholds(data: ThresholdsConfig): void {
|
||||
writeJson('thresholds.json', { ...data, updatedAt: new Date().toISOString() });
|
||||
}
|
||||
|
||||
// ── Alerts ──
|
||||
|
||||
export function getAlerts(): AlertRecord[] {
|
||||
return readJson<AlertRecord[]>('alerts.json', []);
|
||||
}
|
||||
|
||||
export function saveAlerts(alerts: AlertRecord[]): void {
|
||||
// Cap at MAX_ALERTS (FIFO — drop oldest resolved first, then oldest overall)
|
||||
let trimmed = alerts;
|
||||
if (trimmed.length > MAX_ALERTS) {
|
||||
const resolved = trimmed.filter(a => a.resolvedAt !== null);
|
||||
const active = trimmed.filter(a => a.resolvedAt === null);
|
||||
const combined = [...active, ...resolved].slice(0, MAX_ALERTS);
|
||||
trimmed = combined;
|
||||
}
|
||||
writeJson('alerts.json', trimmed);
|
||||
}
|
||||
|
||||
export function addAlert(alert: AlertRecord): void {
|
||||
const alerts = getAlerts();
|
||||
alerts.unshift(alert);
|
||||
saveAlerts(alerts);
|
||||
}
|
||||
|
||||
export function updateAlert(id: string, patch: Partial<AlertRecord>): AlertRecord | null {
|
||||
const alerts = getAlerts();
|
||||
const idx = alerts.findIndex(a => a.id === id);
|
||||
if (idx === -1) return null;
|
||||
alerts[idx] = { ...alerts[idx], ...patch };
|
||||
saveAlerts(alerts);
|
||||
return alerts[idx];
|
||||
}
|
||||
|
||||
export function deleteAlert(id: string): boolean {
|
||||
const alerts = getAlerts();
|
||||
const filtered = alerts.filter(a => a.id !== id);
|
||||
if (filtered.length === alerts.length) return false;
|
||||
saveAlerts(filtered);
|
||||
return true;
|
||||
}
|
||||
|
||||
export function clearResolvedAlerts(): number {
|
||||
const alerts = getAlerts();
|
||||
const active = alerts.filter(a => a.resolvedAt === null);
|
||||
const count = alerts.length - active.length;
|
||||
saveAlerts(active);
|
||||
return count;
|
||||
}
|
||||
|
||||
export function getActiveAlerts(metric?: string): AlertRecord[] {
|
||||
const alerts = getAlerts();
|
||||
return alerts.filter(a => a.resolvedAt === null && (!metric || a.metric === metric));
|
||||
}
|
||||
|
||||
// ── Preferences ──
|
||||
|
||||
const defaultPreferences: NotificationPreferences = {
|
||||
recipients: [
|
||||
{ email: 'nafih@bshtechnologies.in', name: 'Nafih', enabled: true },
|
||||
{ email: 'vivek@bshtechnologies.in', name: 'Vivek', enabled: true },
|
||||
],
|
||||
cooldownMinutes: 15,
|
||||
enabledSeverities: ['critical', 'warning', 'info'],
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
export function getPreferences(): NotificationPreferences {
|
||||
return readJson<NotificationPreferences>('preferences.json', defaultPreferences);
|
||||
}
|
||||
|
||||
export function savePreferences(data: NotificationPreferences): void {
|
||||
writeJson('preferences.json', { ...data, updatedAt: new Date().toISOString() });
|
||||
}
|
||||
36
server/src/services/traffic.ts
Normal file
36
server/src/services/traffic.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
import { sshManager } from '../ssh/client.js';
|
||||
import type { TrafficInfo } from '../types/index.js';
|
||||
|
||||
export async function getTrafficInfo(): Promise<TrafficInfo> {
|
||||
// Run all traffic checks in a single SSH command for efficiency
|
||||
const script = [
|
||||
// 1. Total established TCP connections to ports 80/443
|
||||
`echo "CONN_TOTAL=$(ss -t state established '( dport = :80 or dport = :443 or sport = :80 or sport = :443 )' 2>/dev/null | tail -n +2 | wc -l)"`,
|
||||
|
||||
// 2. Top IP by connection count + its count
|
||||
`TOP=$(ss -t state established '( dport = :80 or dport = :443 or sport = :80 or sport = :443 )' 2>/dev/null | tail -n +2 | awk '{print $5}' | rev | cut -d: -f2- | rev | sort | uniq -c | sort -rn | head -1); echo "TOP_IP_COUNT=$(echo "$TOP" | awk '{print $1}')"; echo "TOP_IP=$(echo "$TOP" | awk '{print $2}')"`,
|
||||
|
||||
// 3. Nginx request rate (requests in last 60 seconds from access log)
|
||||
`if [ -f /var/log/nginx/access.log ]; then CUTOFF=$(date -d '1 minute ago' '+%d/%b/%Y:%H:%M' 2>/dev/null || date -v-1M '+%d/%b/%Y:%H:%M' 2>/dev/null); REQ=$(awk -v cutoff="$CUTOFF" '$4 >= "["cutoff { count++ } END { print count+0 }' /var/log/nginx/access.log 2>/dev/null); echo "REQ_RATE=$REQ"; else echo "REQ_RATE=0"; fi`,
|
||||
|
||||
// 4. 5xx error rate (5xx responses in last 60 seconds)
|
||||
`if [ -f /var/log/nginx/access.log ]; then CUTOFF=$(date -d '1 minute ago' '+%d/%b/%Y:%H:%M' 2>/dev/null || date -v-1M '+%d/%b/%Y:%H:%M' 2>/dev/null); ERR=$(awk -v cutoff="$CUTOFF" '$4 >= "["cutoff && $9 ~ /^5[0-9][0-9]$/ { count++ } END { print count+0 }' /var/log/nginx/access.log 2>/dev/null); echo "ERR_5XX=$ERR"; else echo "ERR_5XX=0"; fi`,
|
||||
].join(' && ');
|
||||
|
||||
const output = await sshManager.execCommand(script);
|
||||
|
||||
// Parse key=value pairs from output
|
||||
const vars = new Map<string, string>();
|
||||
for (const line of output.split('\n')) {
|
||||
const match = line.match(/^(\w+)=(.*)$/);
|
||||
if (match) vars.set(match[1], match[2].trim());
|
||||
}
|
||||
|
||||
return {
|
||||
totalConnections: parseInt(vars.get('CONN_TOTAL') ?? '0', 10) || 0,
|
||||
topIpConnections: parseInt(vars.get('TOP_IP_COUNT') ?? '0', 10) || 0,
|
||||
topIp: vars.get('TOP_IP') ?? 'none',
|
||||
requestsPerMinute: parseInt(vars.get('REQ_RATE') ?? '0', 10) || 0,
|
||||
errors5xxPerMinute: parseInt(vars.get('ERR_5XX') ?? '0', 10) || 0,
|
||||
};
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
// ─── Server Health Monitoring API Types ───
|
||||
export * from './notifications.js';
|
||||
|
||||
export interface SystemInfo {
|
||||
hostname: string;
|
||||
@@ -45,28 +46,12 @@ export interface NginxStatus {
|
||||
checkedAt: string;
|
||||
}
|
||||
|
||||
// ─── User & Notification Types ───
|
||||
|
||||
export type UserRole = 'developer' | 'server-admin';
|
||||
|
||||
export interface MonitorUser {
|
||||
id: string;
|
||||
email: string;
|
||||
name: string;
|
||||
role: UserRole;
|
||||
notifyOnError: boolean;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
export interface Notification {
|
||||
id: string;
|
||||
type: 'error' | 'warning' | 'info' | 'recovery';
|
||||
title: string;
|
||||
message: string;
|
||||
source: string;
|
||||
timestamp: string;
|
||||
read: boolean;
|
||||
emailSent: boolean;
|
||||
export interface TrafficInfo {
|
||||
totalConnections: number;
|
||||
topIpConnections: number;
|
||||
topIp: string;
|
||||
requestsPerMinute: number;
|
||||
errors5xxPerMinute: number;
|
||||
}
|
||||
|
||||
export interface OverviewResponse {
|
||||
@@ -75,6 +60,7 @@ export interface OverviewResponse {
|
||||
disk: DiskInfo | null;
|
||||
docker: DockerContainer[] | null;
|
||||
nginx: NginxStatus | null;
|
||||
traffic: TrafficInfo | null;
|
||||
errors: Record<string, string>;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
60
server/src/types/notifications.ts
Normal file
60
server/src/types/notifications.ts
Normal file
@@ -0,0 +1,60 @@
|
||||
// ─── Notification & Alerting Types ───
|
||||
|
||||
export type AlertSeverity = 'critical' | 'warning' | 'info';
|
||||
|
||||
export type MetricKey =
|
||||
| 'cpu_percent'
|
||||
| 'load_avg_1m'
|
||||
| 'memory_percent'
|
||||
| 'disk_percent'
|
||||
| 'nginx_status'
|
||||
| 'container_stopped'
|
||||
| 'connections_total'
|
||||
| 'connections_per_ip'
|
||||
| 'request_rate'
|
||||
| 'error_rate_5xx';
|
||||
|
||||
export interface ThresholdConfig {
|
||||
metric: MetricKey;
|
||||
warning: number;
|
||||
critical: number;
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
export interface ThresholdsConfig {
|
||||
thresholds: ThresholdConfig[];
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface AlertRecord {
|
||||
id: string;
|
||||
metric: MetricKey;
|
||||
severity: AlertSeverity;
|
||||
message: string;
|
||||
value: number | string;
|
||||
threshold: number | string;
|
||||
firedAt: string;
|
||||
resolvedAt: string | null;
|
||||
emailSent: boolean;
|
||||
acknowledged: boolean;
|
||||
}
|
||||
|
||||
export interface NotificationPreferences {
|
||||
recipients: EmailRecipient[];
|
||||
cooldownMinutes: number;
|
||||
enabledSeverities: AlertSeverity[];
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface EmailRecipient {
|
||||
email: string;
|
||||
name: string;
|
||||
enabled: boolean;
|
||||
}
|
||||
|
||||
export interface NotificationSummary {
|
||||
activeWarnings: number;
|
||||
activeCriticals: number;
|
||||
unacknowledged: number;
|
||||
total: number;
|
||||
}
|
||||
Reference in New Issue
Block a user