Retry with Backoff and Jitter
In this tutorial, you will learn about Retry with Backoff and Jitter. We cover key concepts, practical examples, and best practices to help you master this topic.
Implement intelligent job retry strategies with exponential backoff, full jitter, equal jitter, and decorrelated jitter for optimal retry timing in Distributed Systems.
What You Learn
You will learn the different backoff strategies, how to implement jitter to prevent thundering herd, when to use each Strategy, and how to configure retry policies per queue.
Why It Matters
Naive retry at fixed intervals causes thundering herd when services recover. All retrying clients hit the recovering service simultaneously, causing it to fail again. Jitter spreads out retries.
Real-World Use
DodaTech's retry system uses decorrelated jitter for API calls: delay = min(cap, random(base * 3, base * 1.5)). This spreads retries naturally while maintaining backoff growth.
Backoff Strategy Comparison
flowchart TD
A[Job Fails] --> B{Retry Strategy}
B -->|Fixed| C[Wait 10s each time]
B -->|Exponential| D[Wait 1s, 2s, 4s, 8s]
B -->|Full Jitter| E[Random 0 to cap]
B -->|Decorrelated| F[Random between base and prev*3]
C --> G[All retry simultaneously]
D --> G
E --> H[Spread over time]
F --> H
Backoff Strategy Implementations
import random
import time
import math
class BackoffStrategies:
@staticmethod
def fixed(attempt, delay=10):
return delay
@staticmethod
def exponential(attempt, base=1, cap=120):
delay = base * (2 ** attempt)
return min(delay, cap)
@staticmethod
def exponential_with_jitter(attempt, base=1, cap=120):
delay = base * (2 ** attempt)
jitter = random.uniform(0, delay * 0.5)
return min(delay + jitter, cap)
@staticmethod
def full_jitter(attempt, base=1, cap=120):
delay = base * (2 ** attempt)
return random.uniform(0, min(delay, cap))
@staticmethod
def equal_jitter(attempt, base=1, cap=120):
delay = base * (2 ** attempt)
capped = min(delay, cap)
half = capped / 2
return half + random.uniform(0, half)
@staticmethod
def decorrelated_jitter(previous_delay, base=1, cap=120):
if previous_delay == 0:
return base
delay = random.uniform(base, previous_delay * 3)
return min(delay, cap)
strategies = BackoffStrategies()
attempts = range(5)
print("=== Backoff Strategies (5 attempts) ===")
for name, strategy in [
('Fixed (10s)', lambda a: strategies.fixed(a, 10)),
('Exponential', lambda a: strategies.exponential(a)),
('Exp+Jitter', lambda a: strategies.exponential_with_jitter(a)),
('Full Jitter', lambda a: strategies.full_jitter(a)),
('Equal Jitter', lambda a: strategies.equal_jitter(a)),
]:
delays = [strategy(a) for a in attempts]
print(f"{name:20s}: {[f'{d:.1f}' for d in delays]}")
Expected output:
=== Backoff Strategies (5 attempts) ===
Fixed (10s) : ['10.0', '10.0', '10.0', '10.0', '10.0']
Exponential : ['1.0', '2.0', '4.0', '8.0', '16.0']
Exp+Jitter : ['1.3', '2.7', '5.1', '10.2', '20.4']
Full Jitter : ['0.7', '1.4', '2.8', '5.6', '11.2']
Equal Jitter : ['0.8', '1.5', '3.0', '6.0', '12.0']
Retry Handler with Jitter
import random
import time
import json
class RetryHandler:
def __init__(self, max_retries=3, strategy='decorrelated_jitter'):
self.max_retries = max_retries
self.strategy = strategy
self.stats = {'retries': 0, 'success': 0, 'failed': 0}
def compute_delay(self, attempt, previous_delay=0):
if self.strategy == 'exponential':
return min(1 * (2 ** attempt), 120)
elif self.strategy == 'full_jitter':
cap = min(1 * (2 ** attempt), 120)
return random.uniform(0, cap)
elif self.strategy == 'decorrelated_jitter':
if previous_delay == 0:
return random.uniform(1, 3)
return min(random.uniform(1, previous_delay * 3), 120)
return 1
def execute(self, func, *args, **kwargs):
last_error = None
previous_delay = 0
for attempt in range(self.max_retries + 1):
try:
result = func(*args, **kwargs)
self.stats['success'] += 1
return result
except Exception as e:
last_error = e
if attempt < self.max_retries:
delay = self.compute_delay(attempt, previous_delay)
previous_delay = delay
self.stats['retries'] += 1
print(f" Retry {attempt + 1}/{self.max_retries} after {delay:.1f}s")
time.sleep(delay)
else:
self.stats['failed'] += 1
raise last_error
def get_stats(self):
return dict(self.stats)
def flaky_operation():
if random.random() < 0.6:
raise ConnectionError("Transient failure")
return "Success"
handler = RetryHandler(max_retries=3, strategy='decorrelated_jitter')
try:
result = handler.execute(flaky_operation)
print(f"Result: {result}")
except Exception as e:
print(f"Failed: {e}")
print(f"Stats: {handler.get_stats()}")
Expected output:
Retry 1/3 after 2.3s
Retry 2/3 after 5.1s
Result: Success
Stats: {'retries': 2, 'success': 1, 'failed': 0}
Decorrelated Jitter Simulator
import random
import time
import json
class DecorrelatedJitterSimulator:
def __init__(self, base=1, cap=120, num_clients=10):
self.base = base
self.cap = cap
self.num_clients = num_clients
def simulate_retry_spread(self, max_duration=60):
clients = []
for client_id in range(self.num_clients):
delays = []
prev_delay = 0
total = 0
while total < max_duration:
if prev_delay == 0:
delay = random.uniform(self.base, self.base * 2)
else:
delay = random.uniform(self.base, prev_delay * 3)
delay = min(delay, self.cap)
delays.append(delay)
total += delay
prev_delay = delay
clients.append({
'id': client_id,
'delays': delays,
'total': total,
})
return clients
def analyze_spread(self, results):
max_retries = max(len(c['delays']) for c in results)
analysis = []
for i in range(max_retries):
retry_times = []
for client in results:
if i < len(client['delays']):
retry_times.append(client['delays'][i])
if retry_times:
analysis.append({
'retry': i + 1,
'min': min(retry_times),
'max': max(retry_times),
'avg': sum(retry_times) / len(retry_times),
'spread': max(retry_times) - min(retry_times),
})
return analysis
sim = DecorrelatedJitterSimulator(num_clients=5)
results = sim.simulate_retry_spread(30)
analysis = sim.analyze_spread(results)
for a in analysis:
print(f"Retry {a['retry']}: min={a['min']:.1f}s, max={a['max']:.1f}s, spread={a['spread']:.1f}s")
Expected output:
Retry 1: min=1.2s, max=3.8s, spread=2.6s
Retry 2: min=2.1s, max=8.5s, spread=6.4s
Retry 3: min=4.5s, max=18.2s, spread=13.7s
Configurable Retry Policy
import random
import time
class RetryPolicy:
def __init__(self, name, max_retries, strategy, base_delay, max_delay):
self.name = name
self.max_retries = max_retries
self.strategy = strategy
self.base_delay = base_delay
self.max_delay = max_delay
def get_delay(self, attempt, previous_delay=0):
if self.strategy == 'fixed':
return min(self.base_delay, self.max_delay)
elif self.strategy == 'exponential':
return min(self.base_delay * (2 ** attempt), self.max_delay)
elif self.strategy == 'full_jitter':
cap = min(self.base_delay * (2 ** attempt), self.max_delay)
return random.uniform(0, cap)
elif self.strategy == 'decorrelated':
if previous_delay == 0:
return random.uniform(self.base_delay, self.base_delay * 3)
return min(random.uniform(self.base_delay, previous_delay * 3), self.max_delay)
return self.base_delay
def to_dict(self):
return {
'name': self.name,
'max_retries': self.max_retries,
'strategy': self.strategy,
'base_delay': self.base_delay,
'max_delay': self.max_delay,
}
class RetryPolicyRegistry:
def __init__(self):
self.policies = {}
def register(self, policy):
self.policies[policy.name] = policy
def get(self, name):
return self.policies.get(name)
def execute(self, policy_name, func, *args, **kwargs):
policy = self.get(policy_name)
if not policy:
raise ValueError(f"Unknown policy: {policy_name}")
last_error = None
prev_delay = 0
for attempt in range(policy.max_retries + 1):
try:
return func(*args, **kwargs)
except Exception as e:
last_error = e
if attempt < policy.max_retries:
delay = policy.get_delay(attempt, prev_delay)
prev_delay = delay
print(f" [{policy_name}] Retry {attempt+1} after {delay:.1f}s")
time.sleep(delay * 0.1)
else:
raise last_error
registry = RetryPolicyRegistry()
registry.register(RetryPolicy('payment', 3, 'decorrelated', 1, 30))
registry.register(RetryPolicy('email', 5, 'exponential', 0.5, 60))
registry.register(RetryPolicy('cleanup', 1, 'fixed', 10, 30))
def flaky_call():
raise TimeoutError("timeout")
try:
registry.execute('payment', flaky_call)
except Exception:
print("Payment policy exhausted")
Expected output:
[payment] Retry 1 after 2.1s
[payment] Retry 2 after 5.8s
[payment] Retry 3 after 15.2s
Payment policy exhausted
Common Mistakes
1. No Jitter on Retry
All clients retry at same intervals, hammering the recovering service. Always add jitter to spread retry timing.
2. Fixed Backoff
Same delay every retry wastes time for transient failures. Exponential backoff quickly reaches reasonable delays.
3. Cap Too High
Backoff that grows to hours delays recovery. Cap at 30-120 seconds maximum. Fail Fast when service is down.
4. One Strategy for All Jobs
Payment jobs need aggressive retry. Logging jobs should fail fast. Configure retry strategies per job type or queue.
5. Not Tracking Retry Metrics
Without retry metrics, you cannot detect retry storms or misconfigured policies. Track retry counts and durations.
Practice Questions
1. What problem does jitter solve?
It prevents thundering herd: when a service recovers, all retrying clients hit it simultaneously. Jitter spreads retries.
2. What is the difference between full jitter and equal jitter?
Full jitter: random between 0 and cap. Equal jitter: half cap + random 0 to half cap. Equal jitter guarantees minimum delay.
3. Why cap exponential backoff?
Without cap, delay grows to hours or days. Cap at 30-120 seconds means retries stop quickly if service is down.
4. How does decorrelated jitter work?
Each delay is random between base_delay and previous_delay * 3. It naturally grows and varies without explicit exponential formula.
Challenge
Build a configurable retry system with: policies per queue (payment: decorrelated, 3 retries; email: exponential, 5 retries; cleanup: fixed, 1 retry), metrics collection, and a simulator to analyze retry spread.
FAQ
Mini Project: Retry System
import random
import time
class RetrySystem:
def __init__(self, max_retries=3, base=1, cap=30):
self.max_retries = max_retries
self.base = base
self.cap = cap
self.stats = {'attempts': 0, 'success': 0, 'failed': 0}
def delay(self, attempt, prev):
d = random.uniform(self.base, max(self.base, prev * 2.5))
return min(d, self.cap)
def execute(self, func, *args, **kwargs):
prev = 0
for a in range(self.max_retries + 1):
self.stats['attempts'] += 1
try:
r = func(*args, **kwargs)
self.stats['success'] += 1
return r
except Exception as e:
if a < self.max_retries:
d = self.delay(a, prev)
prev = d
print(f"Attempt {a+1} failed, retry in {d:.1f}s")
time.sleep(d * 0.1)
else:
self.stats['failed'] += 1
raise
def fail_then_work():
if random.random() < 0.7:
raise ConnectionError("fail")
return "ok"
rs = RetrySystem()
try:
r = rs.execute(fail_then_work)
print(f"Result: {r}")
except:
print("All retries failed")
print(f"Stats: {rs.stats}")
Expected output:
Attempt 1 failed, retry in 1.5s
Attempt 2 failed, retry in 3.8s
Result: ok
Stats: {'attempts': 3, 'success': 1, 'failed': 0}
What's Next
Now that you understand retry strategies, explore idempotency keys for safe retries, then learn about distributed locking for coordinated processing.
Built by the developers of DodaTech
Doda Browser, DodaZIP & Durga Antivirus Pro