Load Balancing Algorithms for API Gateways — Round Robin to Consistent Hashing
In this tutorial, you'll learn about Load Balancing Algorithms. We cover key concepts, practical examples, and best practices to help you understand and apply this topic effectively.
Load balancing algorithms at the gateway distribute incoming requests across backend instances to optimize resource utilization, minimize latency, and ensure high availability.
What You'll Learn
By the end of this lesson, you will implement round robin, least connections, IP hash, consistent hashing, and weighted load balancing algorithms, and choose the right Strategy for different use cases.
Why It Matters
The right load balancing algorithm improves performance, prevents overload, and ensures efficient resource utilization across your backend infrastructure.
Real-World Use
Durga Antivirus Pro uses weighted round robin for scan services (heavier instances get more traffic) and consistent hashing for cache-backed report services.
Load Balancing Algorithms
flowchart TD
Request-->Gateway
Gateway-->LB{Load Balancer}
LB-->|Round Robin|RR[Backend 1, 2, 3...]
LB-->|Least Connections|LC[Backend with fewest connections]
LB-->|IP Hash|IP[Backend based on client IP]
LB-->|Consistent Hash|CH[Backend based on key hash]
LB-->|Weighted|W[Backend based on weights]
RR-->Backend[Backend Pool]
LC-->Backend
IP-->Backend
CH-->Backend
W-->Backend
Round Robin Implementation
The simplest algorithm that distributes requests sequentially across backends.
from typing import List, Optional, Dict, Any
import threading
class RoundRobinBalancer:
def __init__(self, backends: List[str]):
self.backends = backends
self.index = 0
self.lock = threading.Lock()
def next_backend(self) -> Optional[str]:
if not self.backends:
return None
with self.lock:
backend = self.backends[self.index]
self.index = (self.index + 1) % len(self.backends)
return backend
def add_backend(self, backend: str):
if backend not in self.backends:
self.backends.append(backend)
def remove_backend(self, backend: str):
if backend in self.backends:
self.backends.remove(backend)
if self.index >= len(self.backends):
self.index = 0
def get_stats(self) -> Dict:
return {
"algorithm": "round_robin",
"backends": list(self.backends),
"current_index": self.index
}
rr = RoundRobinBalancer(["svc-1:8080", "svc-2:8080", "svc-3:8080"])
for i in range(6):
backend = rr.next_backend()
print(f"Request {i+1} -> {backend}")
Least Connections Algorithm
Routes to the backend with the fewest active connections.
from typing import Dict, Optional, List
import threading
class LeastConnectionsBalancer:
def __init__(self):
self.connections: Dict[str, int] = {}
self.lock = threading.Lock()
def add_backend(self, backend: str):
with self.lock:
if backend not in self.connections:
self.connections[backend] = 0
def remove_backend(self, backend: str):
with self.lock:
self.connections.pop(backend, None)
def get_backend(self) -> Optional[str]:
with self.lock:
if not self.connections:
return None
return min(
self.connections,
key=self.connections.get
)
def acquire(self, backend: str):
with self.lock:
if backend in self.connections:
self.connections[backend] += 1
def release(self, backend: str):
with self.lock:
if backend in self.connections:
self.connections[backend] = max(
0, self.connections[backend] - 1
)
def get_stats(self) -> Dict:
with self.lock:
return {
"algorithm": "least_connections",
"connections": dict(self.connections)
}
lc = LeastConnectionsBalancer()
for svc in ["svc-1:8080", "svc-2:8080", "svc-3:8080"]:
lc.add_backend(svc)
lc.acquire("svc-1:8080")
lc.acquire("svc-1:8080")
lc.acquire("svc-2:8080")
for _ in range(4):
backend = lc.get_backend()
print(f"Selected: {backend}")
lc.acquire(backend)
Consistent Hashing
Routes requests to backends based on a hash of the request key, minimizing disruption when backends change.
import hashlib
from typing import Dict, List, Optional, Tuple
import bisect
class ConsistentHashBalancer:
def __init__(self, replicas: int = 150):
self.replicas = replicas
self.ring: Dict[int, str] = {}
self.sorted_keys: List[int] = []
self.nodes: set = set()
def add_node(self, node: str):
self.nodes.add(node)
for i in range(self.replicas):
key = self._hash(f"{node}:{i}")
self.ring[key] = node
self.sorted_keys = sorted(self.ring.keys())
def remove_node(self, node: str):
self.nodes.discard(node)
for i in range(self.replicas):
key = self._hash(f"{node}:{i}")
self.ring.pop(key, None)
self.sorted_keys = sorted(self.ring.keys())
def get_node(self, request_key: str) -> Optional[str]:
if not self.ring:
return None
hash_key = self._hash(request_key)
index = bisect.bisect(self.sorted_keys, hash_key)
if index == len(self.sorted_keys):
index = 0
return self.ring[self.sorted_keys[index]]
def _hash(self, key: str) -> int:
return int(
hashlib.md5(key.encode()).hexdigest(),
16
)
def get_nodes(self) -> List[str]:
return list(self.nodes)
ch = ConsistentHashBalancer(replicas=100)
ch.add_node("cache-1:6379")
ch.add_node("cache-2:6379")
ch.add_node("cache-3:6379")
keys = ["file-1", "file-2", "file-3", "file-4", "file-5"]
for key in keys:
node = ch.get_node(key)
print(f"{key} -> {node}")
ch.remove_node("cache-2:6379")
print("\nAfter removing cache-2:")
for key in keys:
node = ch.get_node(key)
print(f"{key} -> {node}")
Weighted Load Balancing
Distribute traffic proportionally based on backend capacity.
import random
from typing import Dict, List, Optional, Tuple
class WeightedBalancer:
def __init__(self):
self.backends: Dict[str, int] = {}
def add_backend(self, backend: str, weight: int = 1):
self.backends[backend] = weight
def remove_backend(self, backend: str):
self.backends.pop(backend, None)
def get_backend(self) -> Optional[str]:
if not self.backends:
return None
total = sum(self.backends.values())
r = random.randint(1, total)
cumulative = 0
for backend, weight in self.backends.items():
cumulative += weight
if r <= cumulative:
return backend
return None
def update_weight(self, backend: str, weight: int):
if backend in self.backends:
self.backends[backend] = weight
def get_stats(self) -> Dict:
total = sum(self.backends.values())
return {
"algorithm": "weighted",
"backends": {
b: {
"weight": w,
"percentage": round(w / total * 100, 1)
}
for b, w in self.backends.items()
}
}
wb = WeightedBalancer()
wb.add_backend("large-svc:8080", 5)
wb.add_backend("medium-svc:8080", 3)
wb.add_backend("small-svc:8080", 1)
dist = {"large-svc:8080": 0, "medium-svc:8080": 0, "small-svc:8080": 0}
for _ in range(1000):
b = wb.get_backend()
dist[b] += 1
print(f"Distribution: {dist}")
Common Mistakes
Mistake 1: Round Robin Without Health Checks
Round robin continues sending to unhealthy backends. Always combine with health checks.
Mistake 2: Consistent Hashing Too Few Replicas
Less than 100 replicas causes uneven distribution and excessive rebalancing.
Mistake 3: Sticky Sessions Without Weight Awareness
Session affinity combined with weighted routing can overload a single node.
Mistake 4: Ignoring Backend Capacity
Equal distribution to heterogeneous backends wastes capacity on some and overloads others.
Mistake 5: Not Handling Backend Draining
Removing a backend with active connections drops in-flight requests. Implement connection draining.
Practice Questions
- When should you use consistent hashing over round robin?
- How does the least connections algorithm prevent overload?
- What is the impact of adding or removing a node on consistent hashing?
- How do you determine weights for weighted load balancing?
- What is the trade-off between random and round robin distribution?
Challenge
Build a load balancer that supports round robin, least connections, consistent hashing, and weighted distribution, with health check integration that automatically removes unhealthy backends.
FAQ
Mini Project
Build a load balancing module for the gateway that supports round robin, least connections, consistent hashing (with 150 virtual nodes), and weighted distribution, with automatic health checks and dynamic backend addition and removal.
What's Next
Learn about Canary Deployments for safe traffic shifting, or explore Blue-Green Deployments for zero-downtime releases.
Built by the developers of DodaTech
Doda Browser, DodaZIP & Durga Antivirus Pro