Monitoring and Alerting — Complete Implementation Guide
In this tutorial, you will learn about Monitoring and Alerting. We cover key concepts, practical examples, and best practices to help you master this topic.
Monitoring and alerting based on health check data enables automated incident response by tracking health status over time, triggering alerts on failures, and providing dashboards for operational visibility.
What You'll Learn
By the end of this tutorial, you will know how to expose health metrics to Prometheus, build Grafana dashboards, configure alert rules, and integrate health checks with PagerDuty or Slack.
Why It Matters
Health checks without monitoring are just endpoints. Real value comes from tracking health changes over time, alerting on failures, and visualizing system health in dashboards.
Real-World Use
DodaTech's SRE team uses a Grafana dashboard showing all 200 microservice health statuses. Alerts fire when any critical service has been unhealthy for more than 30 seconds.
Monitoring and Alerting Learning Path
flowchart LR
A[Health Aggregation] --> B[Monitoring and Alerting]
B --> C[Prometheus]
B --> D[Grafana]
B --> E[Alerts]
B --> F{You Are Here}
style F fill:#f90,color:#fff
Exposing Health Metrics to Prometheus
Instrument health checks as Prometheus metrics for time-series tracking.
const prometheus = require("prom-client");
class HealthMetricsExporter {
constructor() {
this.healthGauge = new prometheus.Gauge({
name: "service_health_status",
help: "Health status of the service (1=healthy, 0=unhealthy)",
labelNames: ["service", "component"]
});
this.healthCheckDuration = new prometheus.Histogram({
name: "service_health_check_duration_seconds",
help: "Duration of health checks",
labelNames: ["service", "component"],
buckets: [0.01, 0.05, 0.1, 0.5, 1, 5]
});
this.healthCheckTotal = new prometheus.Counter({
name: "service_health_check_total",
help: "Total number of health checks performed",
labelNames: ["service", "result"]
});
}
recordHealth(service, component, healthy, durationMs) {
this.healthGauge.set(
{ service, component },
healthy ? 1 : 0
);
this.healthCheckDuration.observe(
{ service, component },
durationMs / 1000
);
this.healthCheckTotal.inc({
service,
result: healthy ? "success" : "failure"
});
}
}
const metrics = new HealthMetricsExporter();
metrics.recordHealth("user-service", "database", true, 15);
metrics.recordHealth("user-service", "cache", false, 500);
console.log("Prometheus health metrics recorded");
// Metrics are exposed at /metrics endpoint
// # HELP service_health_status Health status of the service (1=healthy, 0=unhealthy)
// # TYPE service_health_status gauge
// service_health_status{service="user-service",component="database"} 1
// service_health_status{service="user-service",component="cache"} 0
Prometheus Alert Rules
Alert rules trigger notifications when health metrics indicate problems.
# prometheus-alerts.yml
groups:
- name: service-health
rules:
- alert: ServiceUnhealthy
expr: service_health_status{component="database"} == 0
for: 30s
labels:
severity: critical
annotations:
summary: "Service {{ $labels.service }} {{ $labels.component }} is unhealthy"
description: "Component {{ $labels.component }} of {{ $labels.service }} has been unhealthy for more than 30 seconds"
- alert: ServiceDegraded
expr: rate(service_health_check_total{result="failure"}[5m]) > 0.1
for: 5m
labels:
severity: warning
annotations:
summary: "Service {{ $labels.service }} has elevated failure rate"
description: "Failure rate > 10% for the last 5 minutes"
- alert: HealthCheckFailing
expr: time() - service_health_check_timestamp > 60
labels:
severity: critical
annotations:
summary: "Health check not reporting for {{ $labels.service }}"
description: "No health check data received for more than 60 seconds"
Grafana Dashboard Configuration
A Grafana dashboard for health check visualization.
class GrafanaDashboardGenerator {
static generateHealthDashboard(title, services) {
return {
title,
panels: services.map((service, i) => ({
title: service,
type: "stat",
gridPos: { x: (i % 4) * 4, y: Math.floor(i / 4) * 3, w: 4, h: 3 },
targets: [{
expr: `service_health_status{service="${service}"}`,
legendFormat: "{{ component }}"
}],
thresholds: {
mode: "absolute",
steps: [
{ color: "red", value: null },
{ color: "green", value: 1 }
]
}
}))
};
}
static generateSummaryRow() {
return {
title: "Health Summary",
type: "row",
panels: [
{
title: "Total Unhealthy",
type: "stat",
targets: [{
expr: "count(service_health_status == 0)"
}]
},
{
title: "Total Healthy",
type: "stat",
targets: [{
expr: "count(service_health_status == 1)"
}]
}
]
};
}
}
const dashboard = GrafanaDashboardGenerator.generateHealthDashboard(
"Microservice Health", ["user-service", "payment-service", "notification-service"]
);
console.log("Grafana dashboard generated with", dashboard.panels.length, "panels");
Webhook Notifications
Send health check alerts to webhook endpoints like Slack or PagerDuty.
class HealthWebhookNotifier {
constructor(webhookUrl) {
this.webhookUrl = webhookUrl;
this.lastNotificationTime = 0;
this.cooldownMs = 60000;
}
async notify(healthResult) {
const now = Date.now();
if (now - this.lastNotificationTime < this.cooldownMs) {
console.log("Skipping notification (cooldown active)");
return;
}
const unhealthyComponents = healthResult.dependencies
?.filter(d => !d.healthy)
?.map(d => d.name) || [];
if (unhealthyComponents.length === 0) return;
const payload = {
text: `Health Alert: ${healthResult.service || "Unknown"}`,
blocks: [
{
type: "header",
text: { type: "plain_text", text: "Health Check Alert" }
},
{
type: "section",
fields: [
{ type: "mrkdwn", text: `*Service:* ${healthResult.service || "N/A"}` },
{ type: "mrkdwn", text: `*Status:* ${healthResult.status}` }
]
},
{
type: "section",
text: {
type: "mrkdwn",
text: `*Unhealthy Components:* ${unhealthyComponents.join(", ") || "None"}`
}
},
{
type: "context",
elements: [{
type: "mrkdwn",
text: `Timestamp: ${new Date().toISOString()}`
}]
}
]
};
try {
const response = await fetch(this.webhookUrl, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(payload)
});
this.lastNotificationTime = now;
console.log("Slack notification sent:", response.status);
} catch (err) {
console.error("Failed to send notification:", err.message);
}
}
}
const slackNotifier = new HealthWebhookNotifier("https://hooks.slack.com/services/TXXXX/BXXXX/XXXXX");
slackNotifier.notify({
service: "user-service",
status: "DOWN",
dependencies: [{ name: "database", healthy: false }]
});
Health Check Logging and Trending
Log health check results for trend analysis and debugging.
class HealthLogger {
constructor() {
this.history = [];
this.maxHistory = 1000;
}
log(result) {
const entry = {
timestamp: new Date().toISOString(),
status: result.status,
healthyCount: result.dependencies?.filter(d => d.healthy).length || 0,
unhealthyCount: result.dependencies?.filter(d => !d.healthy).length || 0,
totalCount: result.dependencies?.length || 0
};
this.history.push(entry);
if (this.history.length > this.maxHistory) {
this.history.shift();
}
if (entry.unhealthyCount > 0) {
console.warn("Health degraded:", JSON.stringify(entry));
} else {
console.log("Health ok:", JSON.stringify(entry));
}
}
getTrend(minutes = 5) {
const cutoff = Date.now() - (minutes * 60000);
const recent = this.history.filter(
e => new Date(e.timestamp).getTime() > cutoff
);
const unhealthyEpisodes = recent.filter(e => e.unhealthyCount > 0);
return {
totalChecks: recent.length,
unhealthyChecks: unhealthyEpisodes.length,
healthPercent: recent.length > 0
? ((recent.length - unhealthyEpisodes.length) / recent.length * 100).toFixed(1)
: 100
};
}
}
const logger = new HealthLogger();
logger.log({ status: "UP", dependencies: [{ healthy: true }] });
logger.log({ status: "DOWN", dependencies: [{ healthy: false }] });
console.log("Trend (5min):", logger.getTrend(5));
// Health ok: {"status":"UP","healthyCount":1,...}
// Health degraded: {"status":"DOWN","healthyCount":0,...}
// Trend (5min): { totalChecks: 2, unhealthyChecks: 1, healthPercent: '50.0' }
Common Mistakes
Not setting up alerts on health check failures -- Health checks are only useful if someone or something responds to failures. Configure alerts for every critical component.
Alert fatigue from flapping health -- A service that fluctuates between healthy and unhealthy generates too many alerts. Use the for: parameter to require sustained failure before alerting.
Not tracking health check history -- Without history, you can't identify trends like gradual degradation. Log all health check results.
Monitoring the health endpoint instead of the metrics -- Checking if /healthz returns 200 doesn't give you trend data. Export Prometheus metrics for dashboards.
Not distinguishing between liveness and readiness alerts -- A liveness failure requires immediate action (restart). A readiness failure is less urgent. Separate alert severities.
Practice Questions
What Prometheus metric type is best for health check status? Gauge. A gauge represents a value that can go up or down, like health check status (1=healthy, 0=unhealthy).
How do you prevent alert fatigue from flapping health checks? Use the for: parameter in Prometheus alert rules to require sustained failure (e.g., for: 30s) before firing.
What is the advantage of tracking health check duration? Increasing health check duration can indicate performance degradation before a complete failure. It's a leading indicator.
Challenge: Implement a health check monitoring system that exposes Prometheus metrics, logs history, and sends Slack alerts on state changes.
class CompleteHealthMonitor {
constructor(slackWebhook) {
this.metrics = new HealthMetricsExporter();
this.logger = new HealthLogger();
this.notifier = new HealthWebhookNotifier(slackWebhook);
this.lastStatus = null;
}
async record(service, components) {
const hasFailure = components.some(c => !c.healthy);
const status = hasFailure ? "DOWN" : "UP";
components.forEach(c => {
this.metrics.recordHealth(service, c.name, c.healthy, c.latencyMs || 0);
});
this.logger.log({ status, dependencies: components });
if (status === "DOWN" && this.lastStatus !== "DOWN") {
await this.notifier.notify({ service, status, dependencies: components });
}
this.lastStatus = status;
return { service, status };
}
}
const monitor = new CompleteHealthMonitor("https://hooks.slack.com/services/...");
monitor.record("api-gateway", [
{ name: "database", healthy: true, latencyMs: 10 },
{ name: "cache", healthy: false, latencyMs: 500 }
]);
FAQ
Mini Project
Build a complete monitoring and alerting system for health checks that exports Prometheus metrics, logs to a time-series database, sends Slack alerts on state changes, and provides a Grafana dashboard configuration.
class MonitoringSystem {
constructor() {
this.exporters = [];
}
addExporter(name, exporter) {
this.exporters.push({ name, exporter });
}
async report(healthResult) {
for (const { name, exporter } of this.exporters) {
try {
await exporter(healthResult);
} catch (err) {
console.error(`Exporter ${name} failed:`, err.message);
}
}
}
}
const system = new MonitoringSystem();
system.addExporter("prometheus", (r) => {
console.log("Exporting to Prometheus:", r.status);
});
system.addExporter("slack", (r) => {
if (r.status === "DOWN") console.log("Sending Slack alert");
});
system.addExporter("logger", (r) => {
console.log("Logging health:", r.status);
});
system.report({ status: "DOWN", service: "test" });
What's Next
Now that you understand monitoring and alerting, build the complete health check project that combines all concepts: endpoints, custom indicators, aggregation, monitoring, and alerting.
Built by the developers of DodaTech
Doda Browser, DodaZIP & Durga Antivirus Pro