Gateway Monitoring with Prometheus and Grafana — Metrics Collection and Visualization
In this tutorial, you'll learn about Gateway Monitoring. We cover key concepts, practical examples, and best practices to help you understand and apply this topic effectively.
Monitoring your API gateway with Prometheus and Grafana provides real-time visibility into request rates, error rates, latency distributions, and resource utilization.
What You'll Learn
By the end of this lesson, you will instrument the gateway with Prometheus metrics, create Grafana dashboards for gateway performance, configure alert rules for anomaly detection, and track SLO Compliance.
Why It Matters
Without monitoring, gateway issues are discovered by users. Proactive monitoring detects degradation before it impacts customers and provides data for capacity planning.
Real-World Use
Durga Antivirus Pro monitors its gateway with Prometheus metrics for request rate, error rate, and P95 latency, visualized on a Grafana dashboard that the operations team reviews continuously.
Monitoring Architecture
flowchart LR
Gateway-->Metrics[/metrics Endpoint]
Metrics-->Prometheus[Prometheus Server]
Prometheus-->Alertmanager[Alertmanager]
Prometheus-->Grafana[Grafana Dashboard]
Alertmanager-->Pager[Pager/Email/Slack]
Prometheus Metrics Instrumentation
Expose gateway metrics at the /metrics endpoint for Prometheus scraping.
from typing import Dict, List, Optional, Callable
import time
import threading
class PrometheusMetrics:
def __init__(self, namespace: str = "gateway"):
self.namespace = namespace
self.counters: Dict[str, int] = {}
self.histograms: Dict[str, List[float]] = {}
self.gauges: Dict[str, float] = {}
self.lock = threading.Lock()
def counter_inc(self, name: str, labels: Dict = None):
with self.lock:
key = self._build_key(name, labels)
self.counters[key] = self.counters.get(key, 0) + 1
def histogram_observe(self, name: str, value: float,
labels: Dict = None):
with self.lock:
key = self._build_key(name, labels)
if key not in self.histograms:
self.histograms[key] = []
self.histograms[key].append(value)
def gauge_set(self, name: str, value: float,
labels: Dict = None):
with self.lock:
key = self._build_key(name, labels)
self.gauges[key] = value
def _build_key(self, name: str,
labels: Dict = None) -> str:
if not labels:
return f"{self.namespace}_{name}"
label_str = "_".join(
f"{k}_{v}" for k, v in sorted(labels.items())
)
return f"{self.namespace}_{name}_{{label_str}}"
def generate_output(self) -> str:
lines = []
with self.lock:
for key, value in self.counters.items():
lines.append(f"# HELP {key} Counter metric")
lines.append(f"# TYPE {key} counter")
lines.append(f"{key} {value}")
for key, values in self.histograms.items():
if not values:
continue
lines.append(f"# HELP {key} Histogram metric")
lines.append(f"# TYPE {key} histogram")
sorted_vals = sorted(values)
count = len(sorted_vals)
sum_vals = sum(sorted_vals)
lines.append(f"{key}_count {count}")
lines.append(f"{key}_sum {sum_vals}")
for p, name in [(50, "0.5"), (90, "0.9"),
(95, "0.95"), (99, "0.99")]:
idx = int(count * p / 100)
val = sorted_vals[min(idx, count - 1)]
lines.append(
f'{key}_{{quantile="{name}"}} {val}'
)
for key, value in self.gauges.items():
lines.append(f"# HELP {key} Gauge metric")
lines.append(f"# TYPE {key} gauge")
lines.append(f"{key} {value}")
return "\n".join(lines)
metrics = PrometheusMetrics()
metrics.counter_inc("requests_total",
{"method": "GET", "path": "/api/scan"})
metrics.histogram_observe("request_duration_ms", 45.2)
metrics.histogram_observe("request_duration_ms", 120.5)
metrics.gauge_set("active_connections", 42)
print(metrics.generate_output())
Grafana Dashboard Configuration
Define dashboard panels for key gateway metrics.
from typing import Dict, List, Any
import json
class GrafanaDashboard:
def __init__(self, title: str):
self.title = title
self.panels: List[Dict] = []
self.current_row = 0
def add_graph_panel(self, title: str,
query: str,
row: int = 0,
span: int = 6):
panel = {
"title": title,
"type": "graph",
"gridPos": {
"h": 8, "w": span,
"x": 0, "y": row
},
"targets": [{
"expr": query,
"legendFormat": "{{method}} {{path}}"
}]
}
self.panels.append(panel)
def add_single_stat(self, title: str,
query: str,
row: int = 0):
panel = {
"title": title,
"type": "stat",
"gridPos": {
"h": 4, "w": 4,
"x": 0, "y": row
},
"targets": [{"expr": query}]
}
self.panels.append(panel)
def add_heatmap(self, title: str,
query: str, row: int = 0):
panel = {
"title": title,
"type": "heatmap",
"gridPos": {
"h": 8, "w": 12,
"x": 0, "y": row
},
"targets": [{"expr": query}]
}
self.panels.append(panel)
def to_json(self) -> str:
dashboard = {
"title": self.title,
"panels": self.panels,
"time": {"from": "now-1h", "to": "now"}
}
return json.dumps(dashboard, indent=2)
dashboard = GrafanaDashboard("API Gateway Overview")
dashboard.add_single_stat(
"Request Rate",
'rate(gateway_requests_total[5m])'
)
dashboard.add_single_stat(
"Error Rate",
'rate(gateway_errors_total[5m])'
)
dashboard.add_single_stat(
"P95 Latency",
'histogram_quantile(0.95, rate(gateway_request_duration_ms_bucket[5m]))'
)
print(dashboard.to_json()[:200] + "...")
Alert Rules
Define Prometheus alerting rules for gateway issues.
from typing import Dict, List
class AlertRule:
def __init__(self, name: str, expr: str,
severity: str = "warning",
duration: str = "5m",
summary: str = "",
description: str = ""):
self.name = name
self.expr = expr
self.severity = severity
self.duration = duration
self.summary = summary
self.description = description
class AlertConfig:
def __init__(self):
self.rules: List[AlertRule] = []
def add_rule(self, rule: AlertRule):
self.rules.append(rule)
def generate_yaml(self) -> str:
groups = [{
"name": "gateway_alerts",
"rules": []
}]
for rule in self.rules:
groups[0]["rules"].append({
"alert": rule.name,
"expr": rule.expr,
"for": rule.duration,
"labels": {"severity": rule.severity},
"annotations": {
"summary": rule.summary,
"description": rule.description
}
})
import yaml
return yaml.dump({"groups": groups},
default_flow_style=False)
alerts = AlertConfig()
alerts.add_rule(AlertRule(
"HighErrorRate",
'rate(gateway_errors_total[5m]) / rate(gateway_requests_total[5m]) > 0.05',
"critical", "5m",
"High gateway error rate",
"Error rate is above 5 percent for 5 minutes"
))
alerts.add_rule(AlertRule(
"HighLatency",
'histogram_quantile(0.95, rate(gateway_request_duration_ms_bucket[5m])) > 1000',
"warning", "10m",
"High gateway latency",
"P95 latency is above 1000ms for 10 minutes"
))
print(f"Configured {len(alerts.rules)} alert rules")
Common Mistakes
Mistake 1: Not Monitoring the Gateway Itself
The gateway is a critical path. If it goes down, all APIs are down. Monitor its health, not just backend health.
Mistake 2: Too Many Metrics
Every metric has a cost. Focus on RED metrics: Rate, Errors, Duration for each endpoint.
Mistake 3: Not Setting Up Alerts
Dashboards without alerts require someone watching constantly. Set up alerting for anomaly detection.
Mistake 4: Ignoring Resource Metrics
Monitor CPU, memory, and Connection Pool usage. Resource exhaustion causes cascading failures.
Mistake 5: No SLO Tracking
Track Service Level Objectives for gateway availability and latency. Alert when approaching the budget.
Practice Questions
- What are the four golden signals of monitoring?
- How does Prometheus scrape metrics from the gateway?
- What is the difference between a counter and a gauge metric?
- How do histogram metrics help with latency analysis?
- What is an SLO and how does it relate to alerting?
Challenge
Build a complete monitoring setup for the gateway that exposes request rate, error rate, and latency histogram metrics at /metrics, configures Prometheus alert rules for high error rate and latency, and generates a Grafana dashboard JSON with request rate, error rate, latency heatmap, and active connections panels.
FAQ
Mini Project
Build a Prometheus metrics exporter for the gateway that exposes request count, error count, and latency histogram with configurable buckets, creates a Grafana dashboard JSON with key panels, and defines Prometheus alert rules for high error rate and latency.
What's Next
Learn about Gateway Alerting for automated Incident Response, or explore Gateway Performance for optimization techniques.
Built by the developers of DodaTech
Doda Browser, DodaZIP & Durga Antivirus Pro