Load Balancing at the API Gateway
In this tutorial, you'll learn about Load Balancing. We cover key concepts, practical examples, and best practices to help you understand and apply this topic effectively.
Load balancing distributes incoming requests across multiple backend instances to ensure no single server is overwhelmed and to provide high availability.
What You'll Learn
By the end of this lesson, you will implement round-robin, least-connections, and IP hash load balancing, configure health checks, and understand weighted distribution.
Why It Matters
Without load balancing, traffic spikes on a single instance cause failures. Proper distribution maximizes throughput and ensures fault tolerance.
Real-World Use
Kong distributes requests across three instances of the user service, and if one instance fails health checks, it is removed from the pool automatically.
Load Balancing Algorithms
flowchart LR
GW[Gateway] -->|Round Robin| S1[Server 1]
GW -->|Least Connections| S2[Server 2]
GW -->|IP Hash| S3[Server 3]
GW -->|Weighted| S4[Server 4]
Round Robin
Distributes requests sequentially across all healthy instances.
# round_robin.py
import json
from typing import List, Optional
class Server:
def __init__(self, name: str, address: str, healthy: bool = True):
self.name = name
self.address = address
self.healthy = healthy
self.active_connections = 0
self.total_requests = 0
class RoundRobinBalancer:
def __init__(self):
self.servers: List[Server] = []
self.current_index = 0
def add_server(self, name: str, address: str):
self.servers.append(Server(name, address))
def mark_health(self, name: str, healthy: bool):
for server in self.servers:
if server.name == name:
server.healthy = healthy
break
def get_next(self) -> Optional[Server]:
healthy_servers = [s for s in self.servers if s.healthy]
if not healthy_servers:
return None
idx = self.current_index % len(healthy_servers)
self.current_index += 1
selected = healthy_servers[idx]
selected.total_requests += 1
return selected
def distribution_report(self) -> dict:
return {
s.name: {"requests": s.total_requests, "healthy": s.healthy}
for s in self.servers
}
balancer = RoundRobinBalancer()
balancer.add_server("app-1", "10.0.0.1:3000")
balancer.add_server("app-2", "10.0.0.2:3000")
balancer.add_server("app-3", "10.0.0.3:3000")
for i in range(6):
server = balancer.get_next()
print(f"Request {i+1}: -> {server.name} ({server.address})")
print(f"\nDistribution: {json.dumps(balancer.distribution_report(), indent=2)}")
Expected output:
Request 1: -> app-1 (10.0.0.1:3000)
Request 2: -> app-2 (10.0.0.2:3000)
Request 3: -> app-3 (10.0.0.3:3000)
Request 4: -> app-1 (10.0.0.1:3000)
Request 5: -> app-2 (10.0.0.2:3000)
Request 6: -> app-3 (10.0.0.3:3000)
Distribution: {
"app-1": {"requests": 2, "healthy": true},
"app-2": {"requests": 2, "healthy": true},
"app-3": {"requests": 2, "healthy": true}
}
Least Connections
Sends requests to the server with the fewest active connections.
# least_connections.py
import json
from typing import List, Optional
class LeastConnectionsBalancer:
def __init__(self):
self.servers: List[Server] = []
def add_server(self, name: str, address: str):
self.servers.append(Server(name, address))
def get_next(self) -> Optional[Server]:
healthy = [s for s in self.servers if s.healthy]
if not healthy:
return None
selected = min(healthy, key=lambda s: s.active_connections)
selected.active_connections += 1
selected.total_requests += 1
return selected
def release(self, server_name: str):
for server in self.servers:
if server.name == server_name and server.active_connections > 0:
server.active_connections -= 1
break
balancer = LeastConnectionsBalancer()
balancer.add_server("app-1", "10.0.0.1:3000")
balancer.add_server("app-2", "10.0.0.2:3000")
balancer.add_server("app-3", "10.0.0.3:3000")
# Simulate some connections already in progress
balancer.servers[0].active_connections = 5
balancer.servers[1].active_connections = 2
for i in range(4):
server = balancer.get_next()
print(f"Request {i+1}: -> {server.name} (connections: {server.active_connections})")
Expected output:
Request 1: -> app-2 (connections: 3)
Request 2: -> app-3 (connections: 1)
Request 3: -> app-2 (connections: 4)
Request 4: -> app-3 (connections: 2)
Health Check Integration
# health_checks.py
import time
from typing import List, Optional
class HealthChecker:
def __init__(self, check_interval: int = 10, timeout: int = 5):
self.check_interval = check_interval
self.timeout = timeout
self.last_checks = {}
def check_server(self, server: Server) -> bool:
import random
healthy = random.random() > 0.3
server.healthy = healthy
self.last_checks[server.name] = {
"timestamp": time.time(),
"healthy": healthy,
}
return healthy
def periodic_check(self, servers: List[Server]):
for server in servers:
if server.name not in self.last_checks:
self.check_server(server)
elif time.time() - self.last_checks[server.name]["timestamp"] > self.check_interval:
self.check_server(server)
checker = HealthChecker(check_interval=10)
servers = [
Server("app-1", "10.0.0.1:3000"),
Server("app-2", "10.0.0.2:3000"),
Server("app-3", "10.0.0.3:3000"),
]
checker.periodic_check(servers)
for server in servers:
status = "healthy" if server.healthy else "unhealthy"
print(f"{server.name}: {status}")
Expected output (example, health varies):
app-1: healthy
app-2: unhealthy
app-3: healthy
Common Mistakes
1. No Health Checks
Without health checks, failed servers remain in the pool and receive traffic, causing errors for clients.
2. Sticky Sessions Without Affinity
When requests from the same client go to different servers, local session data is lost. Use IP hash or sticky cookies.
3. Uneven Weight Distribution
Setting weights without understanding capacity leads to overloaded servers. Base weights on actual server capacity.
4. Ignoring Connection Draining
When removing a server, active connections must finish gracefully. Implement connection draining before shutdown.
5. Single Region Load Balancing
Geographic distribution matters. Use global load balancing (DNS-based or anycast) across regions.
Practice Questions
1. What is the difference between round-robin and least-connections?
Round-robin distributes evenly by count regardless of load. Least-connections sends to the server with fewest active connections.
2. Why are health checks essential for load balancing?
They detect failed servers so the balancer can stop sending traffic to them, preventing errors.
3. What is IP hash load balancing used for?
It ensures requests from the same client always go to the same server, useful for session affinity.
4. How does weighted distribution work?
Each server gets a weight proportional to its capacity. A server with weight 3 gets three times the traffic of a server with weight 1.
Challenge
Implement a weighted round-robin balancer where servers have different capacities and the distribution ratio matches their weights.
FAQ
Mini Project: Weighted Load Balancer
# weighted_balancer.py
import json
from typing import List, Optional
class WeightedServer:
def __init__(self, name: str, address: str, weight: int = 1):
self.name = name
self.address = address
self.weight = weight
self.current_weight = 0
self.total_requests = 0
class WeightedRoundRobin:
def __init__(self):
self.servers: List[WeightedServer] = []
def add_server(self, name: str, address: str, weight: int = 1):
self.servers.append(WeightedServer(name, address, weight))
def get_next(self) -> Optional[WeightedServer]:
if not self.servers:
return None
total = sum(s.weight for s in self.servers)
best = None
for server in self.servers:
server.current_weight += server.weight
if best is None or server.current_weight > best.current_weight:
best = server
if best:
best.current_weight -= total
best.total_requests += 1
return best
wrr = WeightedRoundRobin()
wrr.add_server("large", "10.0.0.1:3000", weight=5)
wrr.add_server("medium", "10.0.0.2:3000", weight=3)
wrr.add_server("small", "10.0.0.3:3000", weight=2)
for i in range(20):
server = wrr.get_next()
print(f"R{i+1:2d}: {server.name:8s} ({server.address})")
print(f"\nTotals: {[(s.name, s.total_requests) for s in wrr.servers]}")
Expected output shows ~50% to large, ~30% to medium, ~20% to small over 20 requests.
What's Next
You understand load balancing. Next, learn about request transformation, then explore response transformation.
Built by the developers of DodaTech
Doda Browser, DodaZIP & Durga Antivirus Pro